cas-bigdatalab/piflow/character-repetition-filter

[
  {"text": "正常文本内容"},
  {"text": "过度重复的文本aaaaaaa..."},
  {"text": "另一段正常文本"}
]

python scripts/run_character_repetition_filter.py \
  --input ./input.json \
  --output ./output.json

python scripts/run_character_repetition_filter.py \
  --input ./input.json \
  --output ./output.json \
  --min_ratio 0.0 \
  --max_ratio 0.4

python scripts/run_character_repetition_filter.py \
  --input ./input.json \
  --output ./output.json \
  --rep_len 5

[OK] Character repetition filtering completed!
   Rep length: 10
   Min ratio: 0.0
   Max ratio: 0.4
   Original documents: 4
   Filtered documents: 2
   Removed documents: 2
   Input file: ./input.json
   Output file: ./output.json

[
  {"text": "过滤后保留的文本1"},
  {"text": "过滤后保留的文本2"}
]

pip install py-data-juicer