87 lines
2.7 KiB
JavaScript
87 lines
2.7 KiB
JavaScript
|
|
import fs from 'fs/promises';
|
|
import path from 'path';
|
|
|
|
const PROJECT_ID = 'poster-product';
|
|
const COLLECTION = 'modelStyles';
|
|
const BASE_URL = `https://firestore.googleapis.com/v1/projects/${PROJECT_ID}/databases/(default)/documents/${COLLECTION}`;
|
|
const OUT_FILE = path.join(process.cwd(), 'data', 'habu_prompts.json');
|
|
|
|
async function fetchAllData() {
|
|
let allDocuments = [];
|
|
let nextPageToken = '';
|
|
let pageCount = 0;
|
|
|
|
console.log(`Starting fetch from ${BASE_URL}...`);
|
|
|
|
do {
|
|
const url = `${BASE_URL}?pageSize=300${nextPageToken ? `&pageToken=${nextPageToken}` : ''}`;
|
|
console.log(`Fetching page ${pageCount + 1}...`);
|
|
|
|
// Node 18+ has fetch
|
|
const response = await fetch(url);
|
|
if (!response.ok) {
|
|
throw new Error(`HTTP error! status: ${response.status} - ${await response.text()}`);
|
|
}
|
|
|
|
const data = await response.json();
|
|
|
|
if (data.documents) {
|
|
allDocuments = allDocuments.concat(data.documents);
|
|
console.log(`Fetched ${data.documents.length} documents. Total: ${allDocuments.length}`);
|
|
} else {
|
|
console.log('No documents in this page.');
|
|
}
|
|
|
|
nextPageToken = data.nextPageToken;
|
|
pageCount++;
|
|
|
|
if (pageCount > 50) {
|
|
console.warn('Reached safety limit of 50 pages. Stopping.');
|
|
break;
|
|
}
|
|
|
|
} while (nextPageToken);
|
|
|
|
console.log('Finished fetching. Transforming data...');
|
|
|
|
const transformedData = allDocuments.map(doc => {
|
|
const fields = doc.fields;
|
|
const obj = {};
|
|
|
|
const extractValue = (field) => {
|
|
if (!field) return null;
|
|
if (field.stringValue !== undefined) return field.stringValue;
|
|
if (field.integerValue !== undefined) return parseInt(field.integerValue);
|
|
if (field.doubleValue !== undefined) return parseFloat(field.doubleValue);
|
|
if (field.booleanValue !== undefined) return field.booleanValue;
|
|
if (field.timestampValue !== undefined) return field.timestampValue;
|
|
if (field.arrayValue !== undefined) {
|
|
return (field.arrayValue.values || []).map((v) => extractValue(v));
|
|
}
|
|
if (field.mapValue !== undefined) {
|
|
const mapObj = {};
|
|
for (const key in field.mapValue.fields) {
|
|
mapObj[key] = extractValue(field.mapValue.fields[key]);
|
|
}
|
|
return mapObj;
|
|
}
|
|
return null;
|
|
};
|
|
|
|
for (const key in fields) {
|
|
obj[key] = extractValue(fields[key]);
|
|
}
|
|
|
|
const nameParts = doc.name.split('/');
|
|
obj.id = nameParts[nameParts.length - 1];
|
|
|
|
return obj;
|
|
});
|
|
|
|
console.log(`Saving ${transformedData.length} items to ${OUT_FILE}...`);
|
|
await fs.writeFile(OUT_FILE, JSON.stringify(transformedData, null, 2));
|
|
console.log('Done.');
|
|
}
|
|
|
|
fetchAllData().catch(console.error);
|