检测乱码的 JavaScript 脚本

Question

我构建了一个应用程序来建议电子邮件地址修复，我需要检测基本上不是真实现有电子邮件地址的电子邮件地址，如下所示：

14370afcdc17429f9e418d5ffbd0334a@magic.com
ce06e817-2149-6cfd-dd24-51b31e93ea1a@stackoverflow.org.il
87c0d782-e09f-056f-f544-c6ec9d17943c@microsoft.org.il
root@ns3160176.ip-151-106-35.eu
ds4-f1g-54-h5-dfg-yk-4gd-htr5-fdg5h@outlook.com
h-rt-dfg4-sv6-fg32-dsv5-vfd5-ds312@gmail.com
test@454-fs-ns-dff4-xhh-43d-frfs.com

我可以进行多次正则表达式检查，但我认为我不会在可疑的“非真实”电子邮件地址中达到良好的百分比，因为我每次都会使用特定的正则表达式模式。

我查看了：
Javascript script to find gibberish words in form inputs
Translate this JavaScript Gibberish please?
Detect keyboard mashed email addresses

最后我查看了这个： Unable to detect gibberish names using Python
我认为它似乎符合我的需求。一个脚本，可以让我对电子邮件地址的每个部分成为乱码（或非真实）电子邮件地址的可能性打分。

所以我想要的是输出：

const strings = ["14370afcdc17429f9e418d5ffbd0334a", "gmail", "ce06e817-2149-6cfd-dd24-51b31e93ea1a", 
                 "87c0d782-e09f-056f-f544-c6ec9d17943c", "space-max", "ns3160176.ip-151-106-35", 
                 "ds4-f1g-54-h5-dfg-yk-4gd-htr5-fdg5h", "outlook", "h-rt-dfg4-sv6-fg32-dsv5-vfd5-
                  ds312", "system-analytics", "454-fs-ns-dff4-xhh-43d-frfs"];

for (let i = 0; i < strings.length; i++) {
   validateGibbrish(strings[i]);
}

而这个validateGibberish函数逻辑会和这个python代码类似：

from nltk.corpus import brown
from collections import Counter
import numpy as np

text = '\n'.join([' '.join([w for w in s]) for s in brown.sents()])

unigrams = Counter(text)
bigrams = Counter(text[i:(i+2)] for i in range(len(text)-2))
trigrams = Counter(text[i:(i+3)] for i in range(len(text)-3))

weights = [0.001, 0.01, 0.989]

def strangeness(text):
    r = 0
    text = '  ' + text + '\n'
    for i in range(2, len(text)):
        char = text[i]
        context1 = text[(i-1):i]
        context2 = text[(i-2):i]
        num = unigrams[char] * weights[0] + bigrams[context1+char] * weights[1] + trigrams[context2+char] * weights[2] 
        den = sum(unigrams.values()) * weights[0] + unigrams[char] + weights[1] + bigrams[context1] * weights[2]
        r -= np.log(num / den)
    return r / (len(text) - 2)

所以最后我会循环所有的字符串并得到这样的结果：

"14370afcdc17429f9e418d5ffbd0334a"                  ->    8.9073
"gmail"                                             ->    1.0044
"ce06e817-2149-6cfd-dd24-51b31e93ea1a"              ->    7.4261
"87c0d782-e09f-056f-f544-c6ec9d17943c"              ->    8.3916
"space-max"                                         ->    1.3553
"ns3160176.ip-151-106-35"                           ->    6.2584
"ds4-f1g-54-h5-dfg-yk-4gd-htr5-fdg5h"               ->    7.1796
"outlook"                                           ->    1.6694
"h-rt-dfg4-sv6-fg32-dsv5-vfd5-ds312"                ->    8.5734
"system-analytics"                                  ->    1.9489
"454-fs-ns-dff4-xhh-43d-frfs"                       ->    7.7058

有没有人提示如何操作并可以提供帮助？
非常感谢:)

更新（2020 年 12 月 22 日）

我设法根据@Konstantin Pribluda 的答案编写了一些代码，香农熵计算：

const getFrequencies = str => {
    let dict = new Set(str);
    return [...dict].map(chr => {
        return str.match(new RegExp(chr, 'g')).length;
    });
};

// Measure the entropy of a string in bits per symbol.
const entropy = str => getFrequencies(str)
    .reduce((sum, frequency) => {
        let p = frequency / str.length;
        return sum - (p * Math.log(p) / Math.log(2));
    }, 0);

const strings = ['14370afcdc17429f9e418d5ffbd0334a', 'or', 'sdf', 'test', 'dave coperfield', 'gmail', 'ce06e817-2149-6cfd-dd24-51b31e93ea1a',
    '87c0d782-e09f-056f-f544-c6ec9d17943c', 'space-max', 'ns3160176.ip-151-106-35',
    'ds4-f1g-54-h5-dfg-yk-4gd-htr5-fdg5h', 'outlook', 'h-rt-dfg4-sv6-fg32-dsv5-vfd5-ds312', 'system-analytics', '454-fs-ns-dff4-xhh-43d-frfs'];

for (let i = 0; i < strings.length; i++) {
    const str = strings[i];
    let result = 0;
    try {
        result = entropy(str);
    }
    catch (error) { result = 0; }
    console.log(`Entropy of '${str}' in bits per symbol:`, result);
}

输出是：

Entropy of '14370afcdc17429f9e418d5ffbd0334a' in bits per symbol: 3.7417292966721747
Entropy of 'or' in bits per symbol: 1
Entropy of 'sdf' in bits per symbol: 1.584962500721156
Entropy of 'test' in bits per symbol: 1.5
Entropy of 'dave coperfield' in bits per symbol: 3.4565647621309536
Entropy of 'gmail' in bits per symbol: 2.3219280948873626
Entropy of 'ce06e817-2149-6cfd-dd24-51b31e93ea1a' in bits per symbol: 3.882021446536749
Entropy of '87c0d782-e09f-056f-f544-c6ec9d17943c' in bits per symbol: 3.787301737252941
Entropy of 'space-max' in bits per symbol: 2.94770277922009
Entropy of 'ns3160176.ip-151-106-35' in bits per symbol: 3.1477803284561103
Entropy of 'ds4-f1g-54-h5-dfg-yk-4gd-htr5-fdg5h' in bits per symbol: 3.3502926596166693
Entropy of 'outlook' in bits per symbol: 2.1280852788913944
Entropy of 'h-rt-dfg4-sv6-fg32-dsv5-vfd5-ds312' in bits per symbol: 3.619340871812292
Entropy of 'system-analytics' in bits per symbol: 3.327819531114783
Entropy of '454-fs-ns-dff4-xhh-43d-frfs' in bits per symbol: 3.1299133176846836

它仍然没有按预期工作，因为“dave coperfield”得到的分数与其他乱码结果大致相同。

还有其他人有更好的逻辑或想法吗？

Answer 1

这是我想出的：

// gibberish detector js
(function (h) {
    function e(c, b, a) { return c < b ? (a = b - c, Math.log(b) / Math.log(a) * 100) : c > a ? (b = c - a, Math.log(100 - a) / Math.log(b) * 100) : 0 } function k(c) { for (var b = {}, a = "", d = 0; d < c.length; ++d)c[d] in b || (b[c[d]] = 1, a += c[d]); return a } h.detect = function (c) {
        if (0 === c.length || !c.trim()) return 0; for (var b = c, a = []; a.length < b.length / 35;)a.push(b.substring(0, 35)), b = b.substring(36); 1 <= a.length && 10 > a[a.length - 1].length && (a[a.length - 2] += a[a.length - 1], a.pop()); for (var b = [], d = 0; d < a.length; d++)b.push(k(a[d]).length); a = 100 * b; for (d = b =
            0; d < a.length; d++)b += parseFloat(a[d], 10); a = b / a.length; for (var f = d = b = 0; f < c.length; f++) { var g = c.charAt(f); g.match(/^[a-zA-Z]+$/) && (g.match(/^(a|e|i|o|u)$/i) && b++, d++) } b = 0 !== d ? b / d * 100 : 0; c = c.split(/[\W_]/).length / c.length * 100; a = Math.max(1, e(a, 45, 50)); b = Math.max(1, e(b, 35, 45)); c = Math.max(1, e(c, 15, 20)); return Math.max(1, (Math.log10(a) + Math.log10(b) + Math.log10(c)) / 6 * 100)
    }
})("undefined" === typeof exports ? this.gibberish = {} : exports)

// email syntax validator
function validateSyntax(email) {
    return /^(([^<>()[\]\\.,;:\s@"]+(\.[^<>()[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/.test(email.toLowerCase());
}

// shannon entropy
function entropy(str) {
    return Object.values(Array.from(str).reduce((freq, c) => (freq[c] = (freq[c] || 0) + 1) && freq, {})).reduce((sum, f) => sum - f / str.length * Math.log2(f / str.length), 0)
}

// vowel counter
function countVowels(word) {
    var m = word.match(/[aeiou]/gi);
    return m === null ? 0 : m.length;
}

// dummy function
function isTrue(value){
    return value
}

// validate string by multiple tests
function detectGibberish(str){
    var strWithoutPunct = str.replace(/[.,\/#!$%\^&\*;:{}=\-_`~()]/g,"");

    var entropyValue = entropy(str) < 3.5;
    var gibberishValue = gibberish.detect(str) < 50;
    var vovelValue = 30 < 100 / strWithoutPunct.length * countVowels(strWithoutPunct) && 100 / strWithoutPunct.length * countVowels(str) < 35;
    return [entropyValue, gibberishValue, vovelValue].filter(isTrue).length > 1
}

// main function
function validateEmail(email) {
    return validateSyntax(email) ? detectGibberish(email.split("@")[0]) : false
}

// tests
document.write(validateEmail("dsfghjdhjs@gmail.com") + "<br/>")
document.write(validateEmail("jhon.smith@gmail.com"))

我结合了多项测试：gibberish-detector.js、香农熵和元音计数（30% 到 35% 之间）。您可以调整一些值以获得更准确的结果。

Answer 2

您可能会考虑做的事情是每次检查每个字符串的随机性，然后根据它们的分数对结果进行排序，并给出一个阈值，排除具有高随机性的那些。难免会错过一些。

有一些检查字符串随机性的实现，例如：

• https://en.wikipedia.org/wiki/Diehard_tests

• http://www.cacert.at/random/

在应用其中一些之前，您可能必须创建一个哈希（将字符和符号映射到整数序列），因为有些仅适用于整数，因为它们测试随机数生成器的属性。

还有一个可以提供帮助的堆栈交换链接是：

• https://stats.stackexchange.com/questions/371150/check-if-a-character-string-is-not-random

PS。我在服务中遇到了类似的问题，因为机器人使用这些类型的虚假电子邮件创建帐户。经过多年处理这个问题（基本上从数据库中手动删除虚假电子邮件），我现在正在考虑在注册页面中引入视觉检查（验证码）以避免挫败感。