Upgrading tokens_replace() to keep tokens and keys togather about quanteda HOT 4 CLOSED

require(quanteda)
txt <- c(d1 = "The United States is bordered by the Atlantic Ocean and the Pacific Ocean.",
         d2 = "The Supreme Court of the United States is seldom in a united state.")
toks <- tokens(txt, remove_punct = TRUE)

dict2 <- dictionary(list(Countries = c("* States", "Federal Republic of *"),
                         Oceans = c("* Ocean")), tolower = FALSE)

tokens_replace2 <- function(x, pattern, replacement = NULL,
                            concatenator = "_", add_key = FALSE) {
    
    x <- tokens_compound(x, pattern, concatenator = concatenator, join = FALSE)
    fixed <- unlist(object2fixed(pattern, types(x), concatenator = concatenator))
    if (add_key) {
        rep <- paste0(fixed, "/", names(fixed))
    } else {
        rep <- names(fixed)
    }
    tokens_replace(x, fixed, rep)
}

tokens_replace2(toks, dict2, concatenator = " ")
#> Tokens consisting of 2 documents.
#> d1 :
#>  [1] "The"       "Countries" "is"        "bordered"  "by"        "the"      
#>  [7] "Oceans"    "and"       "the"       "Oceans"   
#> 
#> d2 :
#>  [1] "The"       "Supreme"   "Court"     "of"        "the"       "Countries"
#>  [7] "is"        "seldom"    "in"        "a"         "united"    "state"
tokens_replace2(toks, dict2, concatenator = " ", add_key = TRUE)
#> Tokens consisting of 2 documents.
#> d1 :
#>  [1] "The"                     "United States/Countries"
#>  [3] "is"                      "bordered"               
#>  [5] "by"                      "the"                    
#>  [7] "Atlantic Ocean/Oceans"   "and"                    
#>  [9] "the"                     "Pacific Ocean/Oceans"   
#> 
#> d2 :
#>  [1] "The"                     "Supreme"                
#>  [3] "Court"                   "of"                     
#>  [5] "the"                     "United States/Countries"
#>  [7] "is"                      "seldom"                 
#>  [9] "in"                      "a"                      
#> [11] "united"                  "state"

tokens_lookup(toks, dict, keep_tokens = TRUE, concatenator = "/")
#> Tokens consisting of 2 documents.
#> d1 :
#>  [1] "The"                     "United States/Countries"
#>  [3] "is"                      "bordered"               
#>  [5] "by"                      "the"                    
#>  [7] "Atlantic Ocean/Oceans"   "and"                    
#>  [9] "the"                     "Pacific Ocean/Oceans"   
#> 
#> d2 :
#>  [1] "The"                     "Supreme"                
#>  [3] "Court"                   "of"                     
#>  [5] "the"                     "United States/Countries"
#>  [7] "is"                      "seldom"                 
#>  [9] "in"                      "a"                      
#> [11] "united"                  "state"

require(quanteda)
#> Loading required package: quanteda
#> Package version: 4.0.0
#> Unicode version: 15.1
#> ICU version: 74.1
#> Parallel computing: 16 of 16 threads used.
#> See https://quanteda.io for tutorials and examples.
txt <- c(d1 = "The United States is bordered by the Atlantic Ocean and the Pacific Ocean.",
         d2 = "The Supreme Court of the United States is seldom in a united state.")
toks <- tokens(txt, remove_punct = TRUE)

dict <- dictionary(list(Countries = c("* States", "Federal Republic of *"),
                         Oceans = c("* Ocean")), tolower = FALSE)

tokens_lookup(toks, dict, exclusive = FALSE, append = TRUE, separator = "/")
#> Tokens consisting of 2 documents.
#> d1 :
#>  [1] "The"                     "United_States/Countries"
#>  [3] "is"                      "bordered"               
#>  [5] "by"                      "the"                    
#>  [7] "Atlantic_Ocean/Oceans"   "and"                    
#>  [9] "the"                     "Pacific_Ocean/Oceans"   
#> 
#> d2 :
#>  [1] "The"                     "Supreme"                
#>  [3] "Court"                   "of"                     
#>  [5] "the"                     "United_States/Countries"
#>  [7] "is"                      "seldom"                 
#>  [9] "in"                      "a"                      
#> [11] "united"                  "state"
tokens_lookup(toks, dict, exclusive = FALSE, append = TRUE, separator = "+")
#> Tokens consisting of 2 documents.
#> d1 :
#>  [1] "The"                     "United_States+Countries"
#>  [3] "is"                      "bordered"               
#>  [5] "by"                      "the"                    
#>  [7] "Atlantic_Ocean+Oceans"   "and"                    
#>  [9] "the"                     "Pacific_Ocean+Oceans"   
#> 
#> d2 :
#>  [1] "The"                     "Supreme"                
#>  [3] "Court"                   "of"                     
#>  [5] "the"                     "United_States+Countries"
#>  [7] "is"                      "seldom"                 
#>  [9] "in"                      "a"                      
#> [11] "united"                  "state"