How many chinese characters do you need to learn to know 99 out of 100 in an average text?

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52

import numpy as np
import plotly.graph_objects as go
import json


def gen_normalization_table():
    return str.maketrans({"麽": "麼", "爲": "為", "爲": "為", "説": "說", "裏": "裡"})


def normalize_chinese(txt, table=gen_normalization_table()):
    return txt.translate(table)


def getFreqDict():
    with open("words.json") as f:
        freqList = json.load(f)

    freqDict = {}
    total = 0

    for x in freqList:
        freq = x["frequency"]
        word = normalize_chinese(x["word"])
        total += freq * len(word)
        for c in word:
            freqDict[c] = freqDict.get(c, 0) + freq

    return {word: freq / total for word, freq in freqDict.items()}


d = getFreqDict()

X = np.arange(1, len(d) + 1)

vals = [val for _, val in sorted(d.items(), key=lambda item: -item[1])]
Y = np.cumsum(np.fromiter(vals, float))


fig = go.Figure()

fig.add_trace(go.Scatter(x=X, y=Y, mode="lines"))

fig.update_layout(
    xaxis=dict(dtick=500, showgrid=True),
    yaxis=dict(dtick=0.10, showgrid=True),
    template="plotly_white",
)

fig.show()

# Export as interactive HTML
fig.write_html("plot.html", full_html=True, include_plotlyjs=True)