【发布时间】：2022-01-05 15:26:07
【问题描述】：
到目前为止，这是我的代码：
const allRows = [];
async function fileToLines(file) {
return new Promise((resolve, reject) => {
reader = new FileReader();
reader.onload = function(e) {
parsedLines = e.target.result.split(/\r|\n|\r\n/);
resolve(parsedLines);
};
reader.readAsText(file);
});
}
document
.getElementById('fileInput')
.addEventListener('change', async function(e) {
var file = e.target.files[0];
if (file != undefined) {
fileToLines(file).then( async id => {
console.log(id)
console.log(parsedLines)
console.log(typeof id);
var idInt = id.map(Number);
var idFiltered = id.filter(function(v){return v!==''});
console.log(idFiltered)
for(let id of idFiltered) {
const row = await getRelease(id);
allRows.push(row);
}
download();
});
}
});
function getRelease(idFiltered) {
return fetch(`https://api.***.com/releases/${idFiltered}`, {
headers: {
'User-Agent': '***/0.1',
},
})
.then(response => response.json())
.then(data => {
if (data.message === 'Release not found.') {
return { error: `Release with ID ${idFiltered} does not exist` };
} else {
const id = data.id;
const delimiter = document.getElementById("delimiter").value || "|";
const artists = data.artists ? data.artists.map(artist => artist.name) : [];
const barcode = data.identifiers.filter(id => id.type === 'Barcode')
.map(barcode => barcode.value);
var formattedBarcode = barcode.join(delimiter);
const country = data.country || 'Unknown';
const genres = data.genres || [];
const formattedGenres = genres.join(delimiter);
const labels = data.labels ? data.labels.map(label => label.name) : [];
const formattedLabels = labels.join(delimiter);
const catno = data.labels ? data.labels.map(catno => catno.catno) : [];
const formattedCatNo = catno.join(delimiter);
const styles = data.styles || [];
const formattedStyles = styles.join(delimiter);
const tracklist = data.tracklist ? data.tracklist
.map(track => track.title) : [];
const formattedTracklist = tracklist.join(delimiter);
const year = data.year || 'Unknown';
const format = data.formats ? data.formats.map(format => format.name) : [];
const qty = data.formats ? data.formats.map(format => format.qty) : [];
const descriptions = data.formats ? data.formats
.map(descriptions => descriptions.descriptions) : [];
const preformattedDescriptions = descriptions.toString()
.replace('"','""').replace(/,/g, ', ');
const formattedDescriptions = '"' + preformattedDescriptions + '"';
return [idFiltered,
artists,
format,
qty,
formattedDescriptions,
formattedLabels,
formattedCatNo,
country,
year,
formattedGenres,
formattedStyles,
formattedBarcode,
formattedTracklist
];
}
});
}
function download() {
const ROW_NAMES = [
"release_id",
"artist",
"format",
"qty",
"format descriptions",
"label",
"catno",
"country",
"year",
"genres",
"styles",
"barcode",
"tracklist"
];
var csvContent = "data:text/csv;charset=utf-8,"
+ ROW_NAMES + "\n" + allRows.map(e => e.join(",")).join("\n");
console.log(csvContent);
var encodedUri = encodeURI(csvContent);
var link = document.createElement("a");
link.setAttribute("href", encodedUri);
link.setAttribute("download", "my_data.csv");
document.body.appendChild(link); // Required for FF
link.click();
}
当我在 2.5 年前（！）之前试图解决这个问题时，有人告诉我最简单的方法是“保持一系列承诺来跟踪请求”，就像这样......
const timer = ms => new Promise(resolve => setTimeout(resolve, ms));
let requests = Promise.resolve();
function getRelease(id) {
const apiCall = requests.then(() =>
fetch(`https://api.***.com/releases/${id}`, {
headers: {
'User-Agent': '***/0.1',
}
})
);
// add to chain / queue
requests = apiCall.then(response =>
+response.headers.get("X-***-Ratelimit-Remaining") <= 1 && timer(60 * 1000)
);
return apiCall
.then(response => response.json())
.then(parseReleaseData);
}
建议此代码的人发表了评论...
现在一个请求将一个接一个地完成，如果达到速率限制，它会等待一分钟。
如果出现速率限制错误，您可能需要重试。您还可以添加多个承诺队列以提高吞吐量。
似乎当我之前尝试过时，它在拨打任何电话之前设置了 60 秒的延迟？我想我想再次尝试这种方法，但我不确定如何编码。就像，我不确定const apiCall = requests.then(() => 是否适合我当前的代码。我可以看到建议的代码实际上返回了“apiCall”，而我的方法设置为返回所有单独的数据字段，所以我不确定如何在那里继续。从主机获取Ratelimit 并根据需要设置超时似乎是一个好方法，但我只是不确定从哪里开始。有什么帮助吗？
编辑：我一直在尝试这样做，但它仍然不起作用：
const timer = ms => new Promise(resolve => setTimeout(resolve, ms));
const createThrottler = (rateLimit) => {
let requestTimestamp = 0;
return (requestHandler) => {
return async (...params) => {
const currentTimestamp = Math.floor(Date.now() / 1000);
if (currentTimestamp < requestTimestamp + rateLimit) {
await timer(rateLimit - (currentTimestamp - requestTimestamp))
}
requestTimestamp = Math.floor(Date.now() / 1000);
return await requestHandler(...params);
}
}
}
const throttle = createThrottler(2500);
const throttleFetch = throttle(fetch);
Edit2：我想知道是否有问题将这行注释掉：
const rateLimit = Math.floor((60 / response.headers.get("X-Discogs-Ratelimit-Remaining")) * 1000);
所以我尝试取消注释它，但现在我明白了
未捕获的引用错误：
response未定义
Edit3：我有一个建议让createThrottler() 函数工作：-
const rateLimit = 2500;
const timer = ms => new Promise(resolve => setTimeout(resolve, ms));
const createThrottler = (rateLimit) => {
let requestTimestamp = 0;
return (requestHandler) => {
return async (...params) => {
const currentTimestamp = Number(Date.now());
if (currentTimestamp < requestTimestamp + rateLimit) {
const timeOut = rateLimit - (currentTimestamp - requestTimestamp);
requestTimestamp = Number(Date.now()) + timeOut;
await timer(timeOut)
}
requestTimestamp = Number(Date.now());
return await requestHandler(...params);
}
}
}
不能说我会为自己解决这个问题，但我们做到了。所以现在我正试图弄清楚如何以及在哪里编码
const rateLimit = Math.floor((60 / response.headers.get("X-Discogs-Ratelimit-Remaining")) * 1000);
没有得到
Uncaught (in promise) ReferenceError: response is not defined
【问题讨论】：
-
使用带有
lodashdebounce的生成器函数。
标签： javascript json csv