Properties
allowedSpecial
allowedSpecial: string[] | "all"
chunkOverlap
chunkOverlap: number
chunkSize
chunkSize: number
disallowedSpecial
disallowedSpecial: string[] | "all"
encodingName
encodingName: TiktokenEncoding
keepSeparator
keepSeparator: boolean
Optional
lengthFunction
lengthFunction?: ((text) => number) | ((text) => Promise<number>)
Type declaration
-
- (text): number
-
Returns number
Type declaration
-
- (text): Promise<number>
-
Returns Promise<number>