delex package

delex.tokenizer module

class delex.tokenizer.AlphaNumericTokenizer

Bases: Tokenizer

NAME = 'alnum_tokens'

class delex.tokenizer.NumericTokenizer

Bases: Tokenizer

NAME = 'num_tokens'

class delex.tokenizer.QGramTokenizer(q, use_freqs: bool = True)

Bases: Tokenizer

class delex.tokenizer.ShingleTokenizer(n)

Bases: Tokenizer

base_tokenize(s: str) → array

class delex.tokenizer.StrippedQGramTokenizer(q, use_freqs: bool = True)

Bases: Tokenizer

RE = re.compile('\\W+')

class delex.tokenizer.StrippedWhiteSpaceTokenizer(use_freqs: bool = True)

Bases: Tokenizer

NAME = 'stripped_whitespace_tokens'

RE = re.compile('[^a-z0-9 ]+')

WHITESPACE_NORM = re.compile('\\s+')

class delex.tokenizer.Tokenizer(use_freqs: bool = True)

Bases: SparkDistributable

class CacheKey(index_col: str, search_col: str | None, tokenizer_type: str)

Bases: CachedObjectKey

index_col: str

search_col: str | None

tokenizer_type: str

build(df: DataFrame, col: str)

deinit()

deinitialize the object, closing resources (e.g. file handles)

init()

initialize the object to be used on in a spark worker

property nunique_tokens

to_spark()

send the obj to the spark cluster to be used on spark workers

tokenize(s: str) → array

tokenize_set(s: str) → array

tokenize the string and return a set or None if the tokenize returns None

tokenize_set_spark(input_col: Column)

return a column expression that gives the same output as the tokenize method. Required for efficiency when building metadata for certain methods

tokenize_spark(input_col: Column)

return a column expression that gives the same output as the tokenize method. Required for efficiency when building metadata for certain methods

class delex.tokenizer.WhiteSpaceTokenizer

Bases: Tokenizer

NAME = 'whitespace_tokens'