{"id":24716,"date":"2025-02-12T11:06:50","date_gmt":"2025-02-12T03:06:50","guid":{"rendered":"http:\/\/139.9.1.231\/?p=24716"},"modified":"2025-02-12T11:06:51","modified_gmt":"2025-02-12T03:06:51","slug":"asr-jiwer","status":"publish","type":"post","link":"http:\/\/139.9.1.231\/index.php\/2025\/02\/12\/asr-jiwer\/","title":{"rendered":"ASR\u8bed\u97f3\u8bc6\u522b\u6307\u6807\u8ba1\u7b97"},"content":{"rendered":"\n<pre class=\"wp-block-code\"><code>#coding=utf-8\r\nimport os\r\nimport sys\r\nimport re\r\nfrom typing import List, Union\r\nimport jiwer\r\nimport pdb\r\n\r\n\r\ndef cal_wer(path_ref, path_hyp, metric_type, output_detail, path_output):\r\n\r\n    ref_text, hyp_text, ref_key = _read_file(path_ref, path_hyp, metric_type)\r\n    \r\n    cal_wer_from_list(ref_text, hyp_text, ref_key, metric_type, output_detail, path_output)\r\n\r\n\r\ndef cal_wer_from_list(\r\n    reference: Union&#91;str, List&#91;str]], \r\n    hypothesis: Union&#91;str, List&#91;str]], \r\n    key: Union&#91;str, List&#91;str]], \r\n    metric_type: str, \r\n    output_detail: bool, \r\n    path_output: str\r\n):\r\n    if isinstance(reference, str):\r\n        reference = &#91;reference]\r\n    if isinstance(hypothesis, str):\r\n        hypothesis = &#91;hypothesis]\r\n    if isinstance(key, str):\r\n        key = &#91;key]\r\n\r\n    # \u6839\u636eref\u662f\u5426\u4e3a\u7a7a, \u5148\u5206\u522b\u8ba1\u7b97wer\u6307\u6807\u518d\u6c47\u603b\r\n    ref_normal, hyp_normal, key_normal = &#91;], &#91;], &#91;]\r\n    ref_empty, hyp_empty, key_empty = &#91;], &#91;], &#91;]\r\n    for i in range(len(reference)):\r\n        if len(reference&#91;i]) != 0:\r\n            ref_normal.append(reference&#91;i])\r\n            hyp_normal.append(hypothesis&#91;i])\r\n            key_normal.append(key&#91;i])\r\n        else:\r\n            ref_empty.append(reference&#91;i])\r\n            hyp_empty.append(hypothesis&#91;i])\r\n            key_empty.append(key&#91;i])\r\n\r\n    res_normal, out_normal = _cal_wer_normal(ref_normal, hyp_normal, metric_type)\r\n    res_empty, out_empty = _cal_wer_empty(hyp_empty, metric_type)\r\n    _summary(ref_normal, hyp_normal, res_normal, out_normal.alignments, key_normal, \r\n             hyp_empty, res_empty, out_empty, key_empty, \r\n             metric_type, output_detail, path_output)\r\n\r\n\r\ndef _read_file(path_ref, path_hyp, metric_type):\r\n    ref_key, ref_text = _preprocess(path_ref, '\\t', metric_type)\r\n    hyp_key, hyp_text = _preprocess(path_hyp, '\\t', metric_type)\r\n\r\n    tmp_dict = {}\r\n    tmp_text = &#91;]\r\n    for i in range(len(hyp_key)):\r\n        if hyp_key&#91;i] not in tmp_dict.keys():\r\n            tmp_dict&#91;hyp_key&#91;i]] = hyp_text&#91;i]\r\n        else:\r\n            print (\"repeated key\")\r\n    for i in range(len(ref_key)):\r\n        if ref_key&#91;i] in tmp_dict.keys():\r\n            tmp_text.append(tmp_dict&#91;ref_key&#91;i]])\r\n        else:\r\n            tmp_text.append(\"\")\r\n\r\n    return ref_text, tmp_text, ref_key\r\n\r\n\r\ndef _preprocess(path_in, sep, metric_type):\r\n    res_key, res_text = &#91;], &#91;]\r\n\r\n    with open(path_in, \"r\", encoding=\"utf-8\") as f_in:\r\n        lines = f_in.readlines()\r\n        for line in lines:\r\n            line = line.strip().split(sep, 1)\r\n            if len(line) == 2:\r\n                key, text = line\r\n                text = re.sub(\"&lt;s>\", \"\", text)\r\n                text = re.sub(\"&lt;\/s>\", \"\", text)\r\n                text = re.sub(\"&lt;unk>\", \"\", text)\r\n                text = re.sub(\"@@ \", \"\", text)\r\n                text = re.sub(\"@ \", \"\", text)\r\n                text = re.sub(\"@@\", \"\", text)\r\n                text = re.sub(\"@\", \"\", text)\r\n                #text = re.sub(\" \", \"\", text)\r\n                text = text.lower()\r\n            else:\r\n                key = line&#91;0]\r\n                text = \"\"\r\n\r\n            text = &#91;x for x in text]\r\n            text_tmp = \"\"\r\n            if metric_type == \"wer\":\r\n                for ch in text:\r\n                    if '\\u4e00' &lt;= ch &lt;= '\\u9fff':\r\n                        text_tmp += \" \" + ch + \" \"\r\n                    else:\r\n                        text_tmp += ch\r\n                text = text_tmp.strip().replace(\"  \", \" \")\r\n            elif metric_type == \"cer\":\r\n                text_tmp = \"\".join(text)\r\n                text = text_tmp.strip().replace(\" \", \"\")\r\n            else:\r\n                assert False\r\n\r\n            res_key.append(key)\r\n            res_text.append(text)\r\n\r\n    return res_key, res_text\r\n\r\n\r\ndef _cal_wer_normal(reference, hypothesis, metric_type):\r\n    if metric_type == \"wer\":\r\n        out = jiwer.process_words(reference=reference, hypothesis=hypothesis)\r\n        ERR = out.wer\r\n    elif metric_type == \"cer\":\r\n        out = jiwer.process_characters(reference=reference, hypothesis=hypothesis)\r\n        ERR = out.cer\r\n    else:\r\n        assert False\r\n\r\n    H = out.hits\r\n    S = out.substitutions\r\n    D = out.deletions\r\n    I = out.insertions\r\n    N = H + S + D\r\n\r\n    res = &#91;ERR, N, S, D, I]\r\n\r\n    return res, out\r\n\r\n\r\ndef _cal_wer_empty(hypothesis, metric_type):\r\n    out = &#91;]\r\n\r\n    I = 0\r\n    for hyp in hypothesis:\r\n        if hyp == \"\":\r\n            i = 0\r\n        else:\r\n            if metric_type == \"wer\":\r\n                i = len(hyp.split(\" \"))\r\n            elif metric_type == \"cer\":\r\n                i = len(hyp)\r\n            else:\r\n                assert False\r\n        I += i\r\n        out.append(i)\r\n\r\n    res = &#91;0, 0, 0, 0, I]\r\n\r\n    return res, out\r\n\r\n\r\ndef _summary(ref_normal, hyp_normal, res_normal, out_normal, key_normal,\r\n             hyp_empty, res_empty, out_empty, key_empty, \r\n             metric_type, output_detail, path_output):\r\n    # wer\/cer\u8ba1\u7b97\r\n    _, N, S, D, I = res_normal\r\n    I += res_empty&#91;-1]\r\n    if N != 0:\r\n        ERR = (S + D + I) \/ N\r\n        SUB = S \/ N\r\n        DEL = D \/ N\r\n        INS = I \/ N\r\n        N_WORD = N\r\n    else:\r\n        if I == 0:\r\n            ERR = 0\r\n        else:\r\n            ERR = 1\r\n        SUB, DEL, INS, N_WORD = 0, 0, I, 0\r\n\r\n    # \u53e5\u51c6\u8ba1\u7b97 + \u8be6\u7ec6\u9519\u8bef\u6307\u6807 + \u8be6\u7ec6\u9519\u8bef\u7edf\u8ba1\r\n    utt_normal, alignments_normal, statistics_normal = _analyse_normal(\r\n        ref_normal, hyp_normal, out_normal, key_normal, metric_type)\r\n    utt_empty, alignments_empty, statistics_empty = _analyse_empty(\r\n        hyp_empty, out_empty, key_empty, metric_type)\r\n\r\n    utt = utt_normal + utt_empty\r\n    alignments = alignments_normal + alignments_empty\r\n    for key in statistics_empty&#91;'insert'].keys():\r\n        if key not in statistics_normal&#91;'insert'].keys():\r\n            statistics_normal&#91;'insert']&#91;key] = statistics_empty&#91;'insert']&#91;key]\r\n        else:\r\n            statistics_normal&#91;'insert']&#91;key] += statistics_empty&#91;'insert']&#91;key]\r\n    N_SENT = len(out_normal) + len(out_empty)\r\n    ACC_UTT = utt \/ N_SENT\r\n    res = &#91;ERR, SUB, DEL, INS, N_WORD, ACC_UTT, N_SENT]\r\n\r\n    _format_output(res, alignments, statistics_normal, metric_type, output_detail, path_output)\r\n\r\n\r\ndef _analyse_normal(ref_normal, hyp_normal, out_normal, key_normal, metric_type):\r\n    utt_normal = 0\r\n    alignments_normal = &#91;]\r\n    statistics_normal = {'substitute' : {}, 'delete' : {}, 'insert' : {}}\r\n\r\n    for i, alignment in enumerate(out_normal):\r\n        err, n_hit, n_sub, n_del, n_ins = 0, 0, 0, 0, 0\r\n        ref_align, hyp_align = \"\", \"\"\r\n        sub_align, del_align, ins_align = \"\", \"\", \"\"\r\n        for j, chunk in enumerate(alignment):\r\n            if (metric_type == \"wer\" and (ref_align != \"\" or hyp_align != \"\")):\r\n                ref_align += \" \"\r\n                hyp_align += \" \"\r\n            if chunk.type == 'equal':\r\n                n_hit += chunk.ref_end_idx - chunk.ref_start_idx\r\n                ref_align += _extract_string(ref_normal&#91;i], chunk.ref_start_idx, chunk.ref_end_idx, metric_type)\r\n                hyp_align += _extract_string(hyp_normal&#91;i], chunk.hyp_start_idx, chunk.hyp_end_idx, metric_type)\r\n\r\n            elif chunk.type == 'substitute':\r\n                err += 1\r\n                n_sub += chunk.ref_end_idx - chunk.ref_start_idx\r\n\r\n                ref_sub = _extract_string(ref_normal&#91;i], chunk.ref_start_idx, chunk.ref_end_idx, metric_type)\r\n                hyp_sub = _extract_string(hyp_normal&#91;i], chunk.hyp_start_idx, chunk.hyp_end_idx, metric_type)\r\n\r\n                ref_align += ref_sub\r\n                hyp_align += hyp_sub\r\n\r\n                key_sub = \"(\" + ref_sub + \") --> (\" + hyp_sub + \")\"\r\n\r\n                sub_align += key_sub + \"\\t\"\r\n\r\n                if key_sub not in statistics_normal&#91;'substitute'].keys():\r\n                    statistics_normal&#91;'substitute']&#91;key_sub] = 1\r\n                else:\r\n                    statistics_normal&#91;'substitute']&#91;key_sub] += 1\r\n\r\n            elif chunk.type == 'delete':\r\n                err += 1\r\n                n_del += chunk.ref_end_idx - chunk.ref_start_idx\r\n\r\n                ref_del = _extract_string(ref_normal&#91;i], chunk.ref_start_idx, chunk.ref_end_idx, metric_type)\r\n                hyp_del = \"*\"\r\n\r\n                ref_align += ref_del\r\n                hyp_align += hyp_del\r\n\r\n                key_del = ref_del\r\n\r\n                del_align += key_del + \"\\t\"\r\n\r\n                if key_del not in statistics_normal&#91;'delete'].keys():\r\n                    statistics_normal&#91;'delete']&#91;key_del] = 1\r\n                else:\r\n                    statistics_normal&#91;'delete']&#91;key_del] += 1\r\n\r\n            elif chunk.type == 'insert':\r\n                err += 1\r\n                n_ins += chunk.hyp_end_idx - chunk.hyp_start_idx\r\n\r\n                ref_ins = \"*\"\r\n                hyp_ins = _extract_string(hyp_normal&#91;i], chunk.hyp_start_idx, chunk.hyp_end_idx, metric_type)\r\n\r\n                ref_align += ref_ins\r\n                hyp_align += hyp_ins\r\n\r\n                key_ins = hyp_ins\r\n\r\n                ins_align += key_ins + \"\\t\"\r\n\r\n                if key_ins not in statistics_normal&#91;'insert'].keys():\r\n                    statistics_normal&#91;'insert']&#91;key_ins] = 1\r\n                else:\r\n                    statistics_normal&#91;'insert']&#91;key_ins] += 1\r\n\r\n            else:\r\n                assert False\r\n\r\n        if err == 0:\r\n            utt_normal += 1\r\n        alignments_normal.append((key_normal&#91;i], ref_align, hyp_align, \r\n                                  sub_align, del_align, ins_align, \r\n                                  n_hit, n_sub, n_del, n_ins))\r\n\r\n    return utt_normal, alignments_normal, statistics_normal\r\n\r\n\r\ndef _analyse_empty(hyp_empty, out_empty, key_empty, metric_type):\r\n    utt_empty = 0\r\n    alignments_empty = &#91;]\r\n    statistics_empty = {'insert' : {}}\r\n\r\n    for i, ins in enumerate(out_empty):\r\n        ref_align, hyp_align = \"\", \"\"\r\n        sub_align, del_align, ins_align = \"\", \"\", \"\"\r\n\r\n        if ins == 0:\r\n            utt_empty += 1\r\n        else:\r\n            ref_ins = \"*\"\r\n            hyp_ins = _extract_string(hyp_empty&#91;i], 0, len(hyp_empty&#91;i]), metric_type)\r\n\r\n            ref_align += ref_ins\r\n            hyp_align += hyp_ins\r\n\r\n            key_ins = hyp_ins\r\n\r\n            ins_align += key_ins + \"\\t\"\r\n\r\n            if key_ins not in statistics_empty&#91;'insert'].keys():\r\n                statistics_empty&#91;'insert']&#91;key_ins] = 1\r\n            else:\r\n                statistics_empty&#91;'insert']&#91;key_ins] += 1\r\n        alignments_empty.append((key_empty&#91;i], ref_align, hyp_align, \r\n                                sub_align, del_align, ins_align, \r\n                                0, 0, 0, ins))\r\n\r\n    return utt_empty, alignments_empty, statistics_empty\r\n\r\n\r\ndef _extract_string(s, begin, end, metric_type):\r\n    res = \"\"\r\n    if metric_type == 'wer':\r\n        res = ' '.join(s.split(' ')&#91;begin:end])\r\n    elif metric_type == 'cer':\r\n        res = s&#91;begin:end]\r\n    else:\r\n        assert False\r\n    return res\r\n\r\n\r\ndef _format_output(res, alignments, statistics, metric_type, output_detail, path_output):\r\n    with open(path_output, \"w\", encoding=\"utf-8\") as f_out:\r\n        if output_detail == True:\r\n            f_out.write(\"-\"*100 + \"\\n\")\r\n            for i, sample in enumerate(alignments):\r\n                key, ref, hyp = sample&#91;0:3]\r\n                sub_align, del_align, ins_align = sample&#91;3:6]\r\n                n_hit, n_sub, n_del, n_ins = sample&#91;6:]\r\n\r\n                f_out.write(\"KEY: \" + key + \"\\n\")\r\n                f_out.write(\"REF: \" + ref + \"\\n\")\r\n                f_out.write(\"HYP: \" + hyp + \"\\n\")\r\n                f_out.write(\"CNT: \" + \"H(\" + str(n_hit) + \") \" + \\\r\n                                      \"S(\" + str(n_sub) + \") \" + \\\r\n                                      \"D(\" + str(n_del) + \") \" + \\\r\n                                      \"I(\" + str(n_ins) + \")\\n\")\r\n                f_out.write(\"SUB: \" + sub_align + \"\\n\")\r\n                f_out.write(\"DEL: \" + del_align + \"\\n\")\r\n                f_out.write(\"INS: \" + ins_align + \"\\n\\n\")\r\n            f_out.write(\"-\"*100 + \"\\n\")\r\n\r\n            f_out.write(\"-\"*100 + \"\\n\")\r\n            lst_sub = list(sorted(statistics&#91;'substitute'].items(), key = lambda x : x&#91;1], reverse=True))\r\n            lst_del = list(sorted(statistics&#91;'delete'].items(), key = lambda x : x&#91;1], reverse=True))\r\n            lst_ins = list(sorted(statistics&#91;'insert'].items(), key = lambda x : x&#91;1], reverse=True))\r\n            f_out.write(\"\\n\u66ff\u6362\u9519\u8bef\u7edf\u8ba1: \\n\")\r\n            for x in lst_sub:\r\n                f_out.write(\"\\t\" + x&#91;0] + \"(\" + str(x&#91;1]) + \")\" + \"\\n\")\r\n            f_out.write(\"\\n\u5220\u9664\u9519\u8bef\u7edf\u8ba1: \\n\")\r\n            for x in lst_del:\r\n                f_out.write(\"\\t\" + x&#91;0] + \"(\" + str(x&#91;1]) + \")\" + \"\\n\")\r\n            f_out.write(\"\\n\u63d2\u5165\u9519\u8bef\u7edf\u8ba1: \\n\")\r\n            for x in lst_ins:\r\n                f_out.write(\"\\t\" + x&#91;0] + \"(\" + str(x&#91;1]) + \")\" + \"\\n\")\r\n            f_out.write(\"-\"*100 + \"\\n\")\r\n\r\n        f_out.write(\"-\"*100 + \"\\n\")\r\n        f_out.write(metric_type.upper() + \": \" + str(round(res&#91;0] * 100.0, 2)) + '%\\n')\r\n        f_out.write(\"WORDS: \" + str(res&#91;4]) + \"\\t\")\r\n        f_out.write(\"SUB: \" + str(round(res&#91;1] * 100.0, 2)) + \"%\\t\")\r\n        f_out.write(\"DEL: \" + str(round(res&#91;2] * 100.0, 2)) + \"%\\t\")\r\n        f_out.write(\"INS: \" + str(round(res&#91;3] * 100.0, 2)) + \"%\\n\")\r\n        f_out.write(\"ACC_UTT: \" + str(round(res&#91;5] * 100.0, 2)) + '%\\t')\r\n        f_out.write(\"SENTS: \" + str(res&#91;6]) + '\\n')\r\n        f_out.write(\"-\"*100 + \"\\n\")\r\n    \r\n    print (metric_type + \" calculation done\")\r\n    print (\"saved to \" + path_output)\r\n\r\n\r\nif __name__ == '__main__':\r\n\r\n    '''\r\n    # example of function cal_wer_from_list\r\n    ref = &#91;\"\u4eca \u5929 \u5929 \u6c14\", \"hello \u6211 ok \u7684\", \"\"]\r\n    hyp = &#91;\"\u4eca \u5929 \u5929\", \"halo \u6211 ok \u7684 \u5440\", \"\u566a \u58f0\"]\r\n    key = &#91;\"000\", \"001\", \"002\"]\r\n    path_output = \".\/example.wer\"\r\n    cal_wer(ref, hyp, key, \"wer\", True, path_output)\r\n\r\n    ref = &#91;\"\u4eca\u5929\u5929\u6c14\", \"hello\u6211ok\u7684\", \"\"]\r\n    hyp = &#91;\"\u4eca\u5929\u5929\", \"halo\u6211ok\u7684\u5440\", \"\u566a\u58f0\"]\r\n    key = &#91;\"000\", \"001\", \"002\"]\r\n    path_output = \".\/example.cer\"\r\n    cal_wer_from_list(ref, hyp, key, \"cer\", True, path_output)\r\n    '''\r\n<\/code><\/pre>\n","protected":false},"excerpt":{"rendered":"","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":[],"categories":[34],"tags":[],"_links":{"self":[{"href":"http:\/\/139.9.1.231\/index.php\/wp-json\/wp\/v2\/posts\/24716"}],"collection":[{"href":"http:\/\/139.9.1.231\/index.php\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"http:\/\/139.9.1.231\/index.php\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"http:\/\/139.9.1.231\/index.php\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"http:\/\/139.9.1.231\/index.php\/wp-json\/wp\/v2\/comments?post=24716"}],"version-history":[{"count":1,"href":"http:\/\/139.9.1.231\/index.php\/wp-json\/wp\/v2\/posts\/24716\/revisions"}],"predecessor-version":[{"id":24717,"href":"http:\/\/139.9.1.231\/index.php\/wp-json\/wp\/v2\/posts\/24716\/revisions\/24717"}],"wp:attachment":[{"href":"http:\/\/139.9.1.231\/index.php\/wp-json\/wp\/v2\/media?parent=24716"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"http:\/\/139.9.1.231\/index.php\/wp-json\/wp\/v2\/categories?post=24716"},{"taxonomy":"post_tag","embeddable":true,"href":"http:\/\/139.9.1.231\/index.php\/wp-json\/wp\/v2\/tags?post=24716"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}