Document
📖 indexGenerator
📂 poetry-source📄 第 1 页
下载索引 下载作者表结构 下载内容表结构 let log = console.log.bind(console) let global_base_path = '/code/project/poetry-source' // 异步请求 let httpRequest = function (url) { return new Promise((resolve, reject) => { var xhr = new XMLHttpRequest(); xhr.open("GET", url, true); xhr.onreadystatechange = function () { if (xhr.readyState == 4) { resolve(JSON.parse(xhr.responseText)); } } xhr.send(); }) } let contentCutIndex = function (source) { if (!source || !source.length) { return 0 } const cutChar = ['，', '。', '？', '！'] let minIdx = source.length for (let i = 0; i { httpRequest(`${global_base_path}/source/${path}`).then(data => { // console.log(path) // console.log(data.length) let currIndex = {} data.map((item, i_idx) => { if (!is_main_index || main_author.includes(item.authorName)) { let dynasty_code = getDictIndex(baseDict.category.DYNASTY, item.dynasty) let author_code = getDictIndex(baseDict.category.AUTHOR, item.authorName) // currIndex[ // `${item.dynasty}-${item.title}-${item.authorName}-${item.content ? item.content[0].substr(0, contentCutIndex(item.content[0])) : ''}` // ] = `${f_idx}-${i_idx}` currIndex[ `${dynasty_code}-${item.title}-${author_code}-${item.content ? item.content[0].substr(0, contentCutIndex(item.content[0])) : ''}` ] = `${f_idx}-${i_idx}` } }) return currIndex }).then(idx_list => { Object.assign(fullIndex, idx_list) dealFileCount++ }) }); } function rebuild() { let result = {} Object.keys(search_data_index).forEach(key => { let metas = key.split('-'); let dynastyCode = metas[0]; let authorCode = metas[2]; metas[0] = base_dict.dynasty.dict[dynastyCode] metas[2] = base_dict.author.dict[authorCode] result[metas.join('-')] = search_data_index[key] }) } // object key value 翻转 const invertKeyValues = obj => Object.keys(obj).reduce((acc, key) => { acc[obj[key]] = key; return acc; }, {}); download.onclick = function () { fullIndex = {} dealFileCount = 0 buildFullIndex() let timer = setInterval(() => { if (dealFileCount !== fileCount) { log("等待解析中..." + dealFileCount + "/" + fileCount) } else { clearInterval(timer); log("已解析完成..." + dealFileCount + "/" + fileCount) // baseDict 翻转 log(baseDict) baseDict.dynasty.dict = invertKeyValues(baseDict.dynasty.dict) baseDict.author.dict = invertKeyValues(baseDict.author.dict) log(baseDict) saveData.setDataConver({ name: `${is_main_index ? 'main' : 'full'}_data_index.js`, data: `search_data_index = ${JSON.stringify(fullIndex)}; base_dict = ${JSON.stringify(baseDict)}; let result = {} Object.keys(search_data_index).forEach(key => { let metas = key.split('-'); let dynastyCode = metas[0]; let authorCode = metas[2]; metas[0] = base_dict.dynasty.dict[dynastyCode] metas[2] = base_dict.author.dict[authorCode] result[metas.join('-')] = search_data_index[key] }) search_data_index = result ` }) } }, 500) // saveData.setDataConver({ name: 'full_data_index.json', data: JSON.stringify(fullIndex, null, 2) }) } function sqlNormalize(inStr) { if (Array.isArray(inStr)) { inStr = inStr.join("\n") } return inStr.replace(/\r?\n/g, "\\n") .replace(/'/g, "\\'") } function sqlBuilder(tableName, titleList, data, defaultMap) { let columnSeg = titleList.map(title => `\`${title}\``).join(', ') return data.map( item => titleList.map( title => item[title] ? `'${sqlNormalize(item[title])}'` : defaultMap[title] ? `'${sqlNormalize(defaultMap[title])}'` : 'null' ).join(', ') ) .map(seg => `insert into ${tableName} (${columnSeg}) values (${seg});`) .join('\n'); } downloadAuthorSql.onclick = function () { let titleList = ['id', 'name', 'dynasty', 'birthYear', 'deathYear', 'desc'] httpRequest('source/作者.json').then(data => { let sql = sqlBuilder('t_author', titleList, data) saveData.setDataConver({ name: `author.sql`, data: sql }) }) } downloadPoetrySql.onclick = function () { let titleList = ['id', 'title', 'authorName', 'authorId', 'dynasty', 'appreciation', 'content', 'comment', 'translation', 'intro', 'foreword', 'annotation', 'kindCN'] let sqlList = [] let sql_file_idx = 0 file_path_idx.forEach((path, f_idx) => { let kindCN = path.substr(0, 1) httpRequest(`${global_base_path}/source/${path}`).then(data => sqlBuilder('t_work', titleList, data, { kindCN: kindCN })) .then(sql => { dealFileCount++ sqlList.push(sql) if (dealFileCount === fileCount) { saveData.setDataConver({ name: `work_${sql_file_idx}.sql`, data: sqlList.join('\n') }) sql_file_idx++ sqlList = [] } }) }); }