cas-bigdatalab/piflow/alphanumeric-filter

[
  {"text": "正常的英文文本内容"},
  {"text": "包含123数字的文本"},
  {"text": "只有符号！！！..."},
  {"text": "混合内容abc123中文"}
]

python scripts/run_alphanumeric_filter.py \
  --input ./input.json \
  --output ./output.json

python scripts/run_alphanumeric_filter.py \
  --input ./input.json \
  --output ./output.json \
  --min_ratio 0.3 \
  --max_ratio 0.8

python scripts/run_alphanumeric_filter.py \
  --input ./input.json \
  --output ./output.json \
  --tokenization True

[OK] Alphanumeric filtering completed!
   Min ratio: 0.25
   Max ratio: 1.0
   Tokenization: False
   Original documents: 6
   Filtered documents: 5
   Removed documents: 1
   Input file: ./input.json
   Output file: ./output.json

[
  {"text": "过滤后保留的文本1"},
  {"text": "过滤后保留的文本2"}
]

pip install py-data-juicer