ab/jobia
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

augementation des ranges

Browse source
This commit is contained in:
el 2025-05-24 02:02:17 +02:00
parent 9653e55453
commit 7878062edc
9 changed files with 159 additions and 197 deletions
Download patch file Download diff file Expand all files Collapse all files

90
backend/dist/controllers/jobIngestionController.js vendored
View file

@ -7,39 +7,79 @@ exports.ingestJobOffers = void 0;
const client_1 = require("@prisma/client");
const FranceTravailService_1 = __importDefault(require("../services/FranceTravailService"));
const prisma = new client_1.PrismaClient();
// Fonction utilitaire pour introduire un délai
const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
const ingestJobOffers = async (req, res) => {
console.log('Ingesting job offers...'); // Log pour vérifier que le fichier TS est utilisé
const BATCH_SIZE = 149; // Limite par requête, -1 pour une plage 0-149
const TOTAL_OFFERS_TO_FETCH = 1000; // Nombre total d'offres que vous voulez essayer d'ingérer
const DELAY_MS = 150; // Délai en ms entre chaque requête pour respecter le rate limit (1000ms/10appels = 100ms, donc 150ms est prudent)
let totalIngestedCount = 0;
let page = 0;
try {
const jobOffers = await FranceTravailService_1.default.getJobOffers({ range: '0-149' });
for (const offre of jobOffers.resultats) {
const mappedOffer = {
id: offre.id,
title: offre.intitule,
description: offre.description,
publicationDate: new Date(offre.dateCreation),
romeCode: offre.romeCode,
romeLabel: offre.romeLibelle,
locationLabel: offre.lieuTravail?.libelle || null,
postalCode: offre.lieuTravail?.codePostal || null,
departmentCode: offre.lieuTravail?.codeDepartement || null,
cityName: offre.lieuTravail?.ville || null,
companyName: offre.entreprise?.nom || null,
contractType: offre.typeContrat,
contractLabel: offre.libelleTypeContrat,
};
await prisma.jobOffer.upsert({
where: { id: mappedOffer.id },
update: mappedOffer,
create: mappedOffer,
});
for (let start = 0; start < TOTAL_OFFERS_TO_FETCH; start += (BATCH_SIZE + 1)) {
const end = start + BATCH_SIZE;
const range = `${start}-${end}`;
console.log(`Workspaceing offers with range: ${range}`);
try {
const jobOffersResponse = await FranceTravailService_1.default.getJobOffers({ range });
const jobOffers = jobOffersResponse.resultats; // Assurez-vous que la réponse contient bien un tableau 'resultats'
if (!jobOffers || jobOffers.length === 0) {
console.log(`No more offers found for range ${range}. Stopping ingestion.`);
break; // Arrête la boucle s'il n'y a plus d'offres
}
for (const offre of jobOffers) {
const mappedOffer = {
id: String(offre.id), // S'assurer que l'ID est une string pour Prisma
title: offre.intitule || 'N/A', // Ajouter des valeurs par défaut pour les champs obligatoires si l'API peut retourner null
description: offre.description || 'N/A',
publicationDate: new Date(offre.dateCreation),
romeCode: offre.romeCode || null,
romeLabel: offre.romeLibelle || null,
locationLabel: offre.lieuTravail?.libelle || null,
postalCode: offre.lieuTravail?.codePostal || null,
departmentCode: offre.lieuTravail?.codeDepartement || null,
cityName: offre.lieuTravail?.ville || null,
companyName: offre.entreprise?.nom || null,
contractType: offre.typeContrat || null,
contractLabel: offre.libelleTypeContrat || null,
};
// Assurer la validité des dates pour Prisma
if (isNaN(mappedOffer.publicationDate.getTime())) {
console.warn(`Invalid dateCreation for offer ID: ${offre.id}. Using current date.`);
mappedOffer.publicationDate = new Date();
}
await prisma.jobOffer.upsert({
where: { id: mappedOffer.id },
update: mappedOffer,
create: mappedOffer,
});
totalIngestedCount++;
}
console.log(`Ingested ${jobOffers.length} offers for range ${range}. Total: ${totalIngestedCount}`);
}
catch (batchError) {
// Gérer les erreurs spécifiques à un appel de lot (ex: rate limit temporaire)
const axiosError = batchError; // Utiliser 'any' pour un accès flexible à la propriété 'response'
console.warn(`Error fetching batch for range ${range}:`, axiosError.response?.data || axiosError.message);
// Si c'est une erreur de rate limit, vous pourriez vouloir augmenter le délai ou réessayer
}
// Introduire un délai avant la prochaine requête pour respecter le rate limit
if (start + (BATCH_SIZE + 1) < TOTAL_OFFERS_TO_FETCH) {
await sleep(DELAY_MS);
}
}
res.status(200).json({
message: 'Job offers ingested successfully',
count: jobOffers.resultats.length,
message: `Job offers ingestion complete. Total offers processed: ${totalIngestedCount}`,
count: totalIngestedCount,
});
}
catch (error) {
console.error('Error ingesting job offers:', error);
console.error('Fatal error during job offers ingestion process:', error);
res.status(500).json({ error: 'Failed to ingest job offers' });
}
finally {
await prisma.$disconnect(); // Déconnecter Prisma après l'opération
}
};
exports.ingestJobOffers = ingestJobOffers;