More mantra things
This commit is contained in:
kngako
1
.gitignore
vendored
1
.gitignore
vendored
@@ -81,3 +81,4 @@ config/production.json
|
||||
database.sqlite
|
||||
.vscode/
|
||||
.DS_Store
|
||||
result/
|
||||
|
||||
@@ -15,7 +15,6 @@
|
||||
"secret" : "don't forget to keep your secrets secret",
|
||||
"port" : 7878,
|
||||
"cookie": {
|
||||
// "domain": "mantra.press"
|
||||
},
|
||||
"compiled-render": false,
|
||||
"domain": "https://mantra.press"
|
||||
|
||||
998
package-lock.json
generated
998
package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@@ -21,10 +21,10 @@
|
||||
"body-parser": "^1.19.1",
|
||||
"config": "^3.3.6",
|
||||
"connect-session-sequelize": "^7.1.2",
|
||||
"excel4node": "^1.7.2",
|
||||
"express": "^4.17.2",
|
||||
"express-fileupload": "^1.4.0",
|
||||
"express-session": "^1.17.2",
|
||||
"fast-xml-parser": "^4.1.3",
|
||||
"mantra-db-models": "git+https://code.sigidli.com/mantra/mantra-db-models.git",
|
||||
"markdown-it": "^13.0.1",
|
||||
"passport": "^0.5.2",
|
||||
|
||||
133
scripts/convert-btc-tex-to-markdown.js
Normal file
133
scripts/convert-btc-tex-to-markdown.js
Normal file
@@ -0,0 +1,133 @@
|
||||
const fs = require('fs');
|
||||
const path = require("path")
|
||||
const { XMLParser } = require("fast-xml-parser")
|
||||
|
||||
const appendOptionalSpace = (nextObject) => {
|
||||
const punctuation = [".", ",", "?", ":", "!"]
|
||||
if (nextObject) {
|
||||
|
||||
const nextString = produceString(nextObject)
|
||||
|
||||
|
||||
if (punctuation.includes(nextString.trim().at(0))) {
|
||||
return ""
|
||||
}
|
||||
if (nextString.startsWith("—")) {
|
||||
return ""
|
||||
}
|
||||
return " "
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
const produceString = (xmlObject, references = [], nextObject = null, level = 1) => {
|
||||
if (Array.isArray(xmlObject)) {
|
||||
const array = xmlObject.map((object, index, array) => {
|
||||
return produceString(object, references, array[index+1])
|
||||
})
|
||||
return array.join("")
|
||||
} else {
|
||||
const keys = Object.keys(xmlObject)
|
||||
// console.log("Keys: ", keys)
|
||||
return keys.map((key, index, array) => {
|
||||
if (key == "component") {
|
||||
return produceString(xmlObject[key].slice(3), references)
|
||||
} else if (key == "header") {
|
||||
return produceString(xmlObject[key], references)
|
||||
} else if (key =="contentMeta") {
|
||||
return produceString(xmlObject[key], references)
|
||||
} else if (key =="titleGroup") {
|
||||
return produceString(xmlObject[key], references)
|
||||
} else if (key =="title") {
|
||||
return `# ${produceString(xmlObject[key], references)}`
|
||||
} else if (key =="body") {
|
||||
return produceString(xmlObject[key], references)
|
||||
} else if (key == "?xmltex") {
|
||||
return `${produceString(xmlObject[key], references)} `
|
||||
} else if (key == "#text") {
|
||||
return xmlObject[key]
|
||||
} else if (key == "section") {
|
||||
return `\n${produceString(xmlObject[key], references, level+1)}`
|
||||
} else if (key == "i") {
|
||||
return ` *${produceString(xmlObject[key], references)}*${appendOptionalSpace(nextObject)}`
|
||||
} else if (key == "p") {
|
||||
return `\n${produceString(xmlObject[key], references)}\n`
|
||||
} else if (key == ":@") {
|
||||
return `${produceString(xmlObject[key], references)}`
|
||||
} else if (key == "exlink") {
|
||||
const chapterNumber = xmlObject[":@"]["@_href"].split(":").at(-1).slice(-2)
|
||||
return ` [${Number(chapterNumber)}](exlink://${xmlObject[":@"]["@_href"]})${appendOptionalSpace(nextObject)}`
|
||||
} else if (key == "url") {
|
||||
return ` [${produceString(xmlObject["url"], references)}][${xmlObject[":@"]["@_href"]}] `
|
||||
} else if (key == "link") {
|
||||
return ` [${xmlObject[":@"]["@_href"]}](link://${xmlObject[":@"]["@_href"]})${appendOptionalSpace(nextObject)}`
|
||||
} else if (key == "note") {
|
||||
const footnote = `\n[^${xmlObject[":@"]['@_xml:id']}]: ${produceString(xmlObject[key], references)}\n`
|
||||
references.push(footnote)
|
||||
return `[^${xmlObject[":@"]['@_xml:id']}]${appendOptionalSpace(nextObject)}`
|
||||
} else if (key == "figure") {
|
||||
const image = xmlObject[key].find(o => o.mediaResource)
|
||||
const caption = xmlObject[key].find(o => o.caption)
|
||||
|
||||
if (caption) {
|
||||
return `\n${produceString(image, references)}\n<figure>\n${produceString(caption)}\n</figure>\n`
|
||||
} else {
|
||||
return `\n${produceString(image, references)}\n`
|
||||
}
|
||||
} else if (key == "mediaResource") {
|
||||
return `![](${xmlObject[":@"]["@_href"]})`
|
||||
} else if (key == "caption") {
|
||||
return `<figcaption>${produceString(xmlObject[key], references)}</figcaption>`
|
||||
}
|
||||
return ""
|
||||
}).join("")
|
||||
}
|
||||
}
|
||||
|
||||
const btcRootDir = "/home/sigidli/Documents/business/exonumia/bitcoin-standard/9781119473862"
|
||||
const excludeRootContent = [
|
||||
"pdf",
|
||||
"Manifest",
|
||||
"metadata",
|
||||
"cover_US",
|
||||
"control",
|
||||
"summary.xls",
|
||||
"protocol.html",
|
||||
// "fmatter",
|
||||
// "bmatter"
|
||||
]
|
||||
const rootDirectories = fs.readdirSync(btcRootDir).filter(d => !excludeRootContent.includes(d)) // .filter(d => d == "ch03")
|
||||
|
||||
rootDirectories.forEach(directory => {
|
||||
console.log(directory)
|
||||
const contentDirectories = fs.readdirSync(path.join(btcRootDir, directory))
|
||||
|
||||
// TODO: image export
|
||||
const textFiles = fs.readdirSync(path.join(btcRootDir, `${directory}/text_s`))
|
||||
|
||||
textFiles.forEach(file => {
|
||||
const fileContent = fs.readFileSync(path.join(btcRootDir, `${directory}/text_s/${file}`))
|
||||
const fileStringContent = fileContent.toString().split("\n").slice(6).join("\n")
|
||||
|
||||
const parser = new XMLParser({
|
||||
ignoreAttributes: false,
|
||||
preserveOrder: true,
|
||||
});
|
||||
|
||||
let xmlObject = parser.parse(fileStringContent)
|
||||
const references = []
|
||||
var markdownContent = produceString(xmlObject, references)
|
||||
markdownContent += references.join("")
|
||||
|
||||
const resultDIR = "result"
|
||||
if (!fs.existsSync(resultDIR)) {
|
||||
fs.mkdirSync(resultDIR)
|
||||
}
|
||||
|
||||
fs.writeFileSync(
|
||||
path.join(resultDIR, `${file}.md`),
|
||||
markdownContent
|
||||
)
|
||||
})
|
||||
});
|
||||
|
||||
@@ -1569,65 +1569,73 @@ module.exports = function (options) {
|
||||
for (const sheetName in workbook.Sheets) {
|
||||
const sheet = workbook.Sheets[sheetName];
|
||||
|
||||
const artifactVersionId = sheet["D4"].v
|
||||
const translationArtifactVersionId = sheet["E4"].v
|
||||
if (sheet["E4"]) {
|
||||
const translationArtifactVersionId = sheet["E4"].v
|
||||
|
||||
var lastTranslationChunkId = null
|
||||
var i = 6
|
||||
do {
|
||||
lastTranslationChunkId = null
|
||||
const translationChunkCell = sheet[`D${i}`]
|
||||
if (translationChunkCell) {
|
||||
lastTranslationChunkId = translationChunkCell.v
|
||||
const translatedCell = sheet[`C${i}`]
|
||||
var lastTranslationChunkId = null
|
||||
var i = 6
|
||||
do {
|
||||
lastTranslationChunkId = null
|
||||
const translationChunkCell = sheet[`D${i}`]
|
||||
if (translationChunkCell) {
|
||||
lastTranslationChunkId = translationChunkCell.v
|
||||
const translatedCell = sheet[`C${i}`]
|
||||
|
||||
if (translatedCell) {
|
||||
const translatedText = translatedCell.v
|
||||
if (translatedCell) {
|
||||
const translatedText = translatedCell.v
|
||||
|
||||
if (translatedText) {
|
||||
// TODO: Get translationChunk and create translation...
|
||||
const translationChunk = await db.TranslationChunk.findByPk(lastTranslationChunkId, {
|
||||
include: [
|
||||
{
|
||||
association: db.TranslationChunk.Translation
|
||||
if (translatedText) {
|
||||
// TODO: Get translationChunk and create translation...
|
||||
const translationChunk = await db.TranslationChunk.findByPk(lastTranslationChunkId, {
|
||||
include: [
|
||||
{
|
||||
association: db.TranslationChunk.Translation
|
||||
}
|
||||
]
|
||||
}).catch(error => {
|
||||
console.error("Couldn't process: ",translationChunkCell);
|
||||
console.error(`Error ${lastTranslationChunkId}: `, error);
|
||||
return null;
|
||||
})
|
||||
|
||||
if (translationChunk) {
|
||||
// Create or update translation
|
||||
if (translationChunk.translation) {
|
||||
// Update translation
|
||||
if (translationChunk.translation.text != translatedText) {
|
||||
translationChunk.translation.text = translatedText
|
||||
// TODO: Update who made the update??
|
||||
await translationChunk.translation.save()
|
||||
}
|
||||
} else {
|
||||
// Create translation for this chunk
|
||||
const translation = await db.Translation.create({
|
||||
creatorId: request.user.id,
|
||||
translationChunkId: translationChunk.id,
|
||||
text: translatedText,
|
||||
translationArtifactVersionId: translationArtifactVersionId
|
||||
})
|
||||
}
|
||||
]
|
||||
})
|
||||
|
||||
if (translationChunk) {
|
||||
// Create or update translation
|
||||
if (translationChunk.translation) {
|
||||
// Update translation
|
||||
if (translationChunk.translation.text != translatedText) {
|
||||
translationChunk.translation.text = translatedText
|
||||
// TODO: Update who made the update??
|
||||
await translationChunk.translation.save()
|
||||
}
|
||||
} else {
|
||||
// Create translation for this chunk
|
||||
const translation = await db.Translation.create({
|
||||
creatorId: request.user.id,
|
||||
translationChunkId: translationChunk.id,
|
||||
text: translatedText,
|
||||
translationArtifactVersionId: translationArtifactVersionId
|
||||
})
|
||||
}
|
||||
}
|
||||
} else {
|
||||
console.log(`${sheetName}: Couldn't find a translated cell C${i}: `, translatedCell)
|
||||
console.log("Chunk Id: ", lastTranslationChunkId)
|
||||
}
|
||||
} else {
|
||||
console.log(`${sheetName}: Couldn't find a translated cell C${i}: `, translatedCell)
|
||||
console.log("Chunk Id: ", lastTranslationChunkId)
|
||||
|
||||
|
||||
i++
|
||||
}
|
||||
|
||||
|
||||
i++
|
||||
}
|
||||
} while (lastTranslationChunkId);
|
||||
|
||||
} while (lastTranslationChunkId);
|
||||
|
||||
// Chunks...
|
||||
// Chunks...
|
||||
|
||||
console.log("Done processing all...")
|
||||
console.log("Done processing all...")
|
||||
} else {
|
||||
console.log("Couldn't process sheet: ", sheetName)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
response.redirect(`/projects/${project.id}`)
|
||||
|
||||
@@ -320,7 +320,7 @@ module.exports = function (options) {
|
||||
}).then(translationChapter => {
|
||||
if (translationChapter) {
|
||||
response.contentType = "text/plain"
|
||||
response.send(markdownProducer.produceMarkdownString(tokens))
|
||||
response.send(markdownProducer.produceMarkdownString(translationChapter))
|
||||
} else {
|
||||
next()
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user