# Buildsheet autogenerated by ravenadm tool -- Do not edit.

NAMEBASE=		R-tokenizers
VERSION=		0.3.0
KEYWORDS=		cran
VARIANTS=		standard
SDESC[standard]=	Fast tokenization of natural language text
HOMEPAGE=		https://docs.ropensci.org/tokenizers/
CONTACT=		CRAN_Automaton[cran@ironwolf.systems]

DOWNLOAD_GROUPS=	main
SITES[main]=		CRAN/src/contrib
			https://loki.dragonflybsd.org/cranfiles/
DISTFILE[1]=		tokenizers_0.3.0.tar.gz:main
DIST_SUBDIR=		CRAN
DF_INDEX=		1
SPKGS[standard]=	single

OPTIONS_AVAILABLE=	none
OPTIONS_STANDARD=	none

BUILDRUN_DEPENDS=	R-stringi:single:standard
			R-Rcpp:single:standard
			R-SnowballC:single:standard

USES=			cran gmake

DISTNAME=		tokenizers

GENERATED=		yes

INSTALL_REQ_TOOLCHAIN=	yes

[FILE:608:descriptions/desc.single]
tokenizers: Fast, Consistent Tokenization of Natural Language Text

Convert natural language text into tokens. Includes tokenizers for shingled
n-grams, skip n-grams, words, word stems, sentences, paragraphs,
characters, shingled characters, lines, Penn Treebank, regular expressions,
as well as functions for counting characters, words, and sentences, and a
function for splitting longer texts into separate documents, each with the
same number of words. The tokenizers have a consistent interface, and the
package is built on the 'stringi' and 'Rcpp' packages for fast yet correct
tokenization in 'UTF-8'.


[FILE:107:distinfo]
24571e4642a1a2d9f4f4c7a363b514eece74788d59c09012a5190ee718a91c29       458876 CRAN/tokenizers_0.3.0.tar.gz