From 7878062edcf2bb43efb81adc7229a93e5ff91657 Mon Sep 17 00:00:00 2001 From: el Date: Sat, 24 May 2025 02:02:17 +0200 Subject: [PATCH] augementation des ranges --- .gitignore | 1 + .../controllers/jobIngestionController.js | 90 +++++++++++----- backend/dist/index.js | 2 + backend/package-lock.json | 8 +- backend/package.json | 16 +-- .../src/controllers/jobIngestionController.js | 55 ---------- .../src/controllers/jobIngestionController.ts | 101 +++++++++++++----- .../src/controllers/jobSearchController.js | 78 -------------- backend/src/index.ts | 5 +- 9 files changed, 159 insertions(+), 197 deletions(-) delete mode 100644 backend/src/controllers/jobIngestionController.js delete mode 100644 backend/src/controllers/jobSearchController.js diff --git a/.gitignore b/.gitignore index 055c6de..46aceda 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .env /documentation/ +node_modules diff --git a/backend/dist/controllers/jobIngestionController.js b/backend/dist/controllers/jobIngestionController.js index a723b6b..c0e433a 100644 --- a/backend/dist/controllers/jobIngestionController.js +++ b/backend/dist/controllers/jobIngestionController.js @@ -7,39 +7,79 @@ exports.ingestJobOffers = void 0; const client_1 = require("@prisma/client"); const FranceTravailService_1 = __importDefault(require("../services/FranceTravailService")); const prisma = new client_1.PrismaClient(); +// Fonction utilitaire pour introduire un délai +const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms)); const ingestJobOffers = async (req, res) => { + console.log('Ingesting job offers...'); // Log pour vérifier que le fichier TS est utilisé + const BATCH_SIZE = 149; // Limite par requête, -1 pour une plage 0-149 + const TOTAL_OFFERS_TO_FETCH = 1000; // Nombre total d'offres que vous voulez essayer d'ingérer + const DELAY_MS = 150; // Délai en ms entre chaque requête pour respecter le rate limit (1000ms/10appels = 100ms, donc 150ms est prudent) + let totalIngestedCount = 0; + let page = 0; try { - const jobOffers = await FranceTravailService_1.default.getJobOffers({ range: '0-149' }); - for (const offre of jobOffers.resultats) { - const mappedOffer = { - id: offre.id, - title: offre.intitule, - description: offre.description, - publicationDate: new Date(offre.dateCreation), - romeCode: offre.romeCode, - romeLabel: offre.romeLibelle, - locationLabel: offre.lieuTravail?.libelle || null, - postalCode: offre.lieuTravail?.codePostal || null, - departmentCode: offre.lieuTravail?.codeDepartement || null, - cityName: offre.lieuTravail?.ville || null, - companyName: offre.entreprise?.nom || null, - contractType: offre.typeContrat, - contractLabel: offre.libelleTypeContrat, - }; - await prisma.jobOffer.upsert({ - where: { id: mappedOffer.id }, - update: mappedOffer, - create: mappedOffer, - }); + for (let start = 0; start < TOTAL_OFFERS_TO_FETCH; start += (BATCH_SIZE + 1)) { + const end = start + BATCH_SIZE; + const range = `${start}-${end}`; + console.log(`Workspaceing offers with range: ${range}`); + try { + const jobOffersResponse = await FranceTravailService_1.default.getJobOffers({ range }); + const jobOffers = jobOffersResponse.resultats; // Assurez-vous que la réponse contient bien un tableau 'resultats' + if (!jobOffers || jobOffers.length === 0) { + console.log(`No more offers found for range ${range}. Stopping ingestion.`); + break; // Arrête la boucle s'il n'y a plus d'offres + } + for (const offre of jobOffers) { + const mappedOffer = { + id: String(offre.id), // S'assurer que l'ID est une string pour Prisma + title: offre.intitule || 'N/A', // Ajouter des valeurs par défaut pour les champs obligatoires si l'API peut retourner null + description: offre.description || 'N/A', + publicationDate: new Date(offre.dateCreation), + romeCode: offre.romeCode || null, + romeLabel: offre.romeLibelle || null, + locationLabel: offre.lieuTravail?.libelle || null, + postalCode: offre.lieuTravail?.codePostal || null, + departmentCode: offre.lieuTravail?.codeDepartement || null, + cityName: offre.lieuTravail?.ville || null, + companyName: offre.entreprise?.nom || null, + contractType: offre.typeContrat || null, + contractLabel: offre.libelleTypeContrat || null, + }; + // Assurer la validité des dates pour Prisma + if (isNaN(mappedOffer.publicationDate.getTime())) { + console.warn(`Invalid dateCreation for offer ID: ${offre.id}. Using current date.`); + mappedOffer.publicationDate = new Date(); + } + await prisma.jobOffer.upsert({ + where: { id: mappedOffer.id }, + update: mappedOffer, + create: mappedOffer, + }); + totalIngestedCount++; + } + console.log(`Ingested ${jobOffers.length} offers for range ${range}. Total: ${totalIngestedCount}`); + } + catch (batchError) { + // Gérer les erreurs spécifiques à un appel de lot (ex: rate limit temporaire) + const axiosError = batchError; // Utiliser 'any' pour un accès flexible à la propriété 'response' + console.warn(`Error fetching batch for range ${range}:`, axiosError.response?.data || axiosError.message); + // Si c'est une erreur de rate limit, vous pourriez vouloir augmenter le délai ou réessayer + } + // Introduire un délai avant la prochaine requête pour respecter le rate limit + if (start + (BATCH_SIZE + 1) < TOTAL_OFFERS_TO_FETCH) { + await sleep(DELAY_MS); + } } res.status(200).json({ - message: 'Job offers ingested successfully', - count: jobOffers.resultats.length, + message: `Job offers ingestion complete. Total offers processed: ${totalIngestedCount}`, + count: totalIngestedCount, }); } catch (error) { - console.error('Error ingesting job offers:', error); + console.error('Fatal error during job offers ingestion process:', error); res.status(500).json({ error: 'Failed to ingest job offers' }); } + finally { + await prisma.$disconnect(); // Déconnecter Prisma après l'opération + } }; exports.ingestJobOffers = ingestJobOffers; diff --git a/backend/dist/index.js b/backend/dist/index.js index ddd9542..8744c32 100644 --- a/backend/dist/index.js +++ b/backend/dist/index.js @@ -7,12 +7,14 @@ const express_1 = __importDefault(require("express")); const dotenv_1 = __importDefault(require("dotenv")); const jobIngestionRoutes_1 = __importDefault(require("./routes/jobIngestionRoutes")); const jobSearchRoutes_1 = __importDefault(require("./routes/jobSearchRoutes")); +const jobIngestionController_1 = require("./controllers/jobIngestionController"); dotenv_1.default.config(); const app = (0, express_1.default)(); const PORT = process.env.PORT || 3000; app.use(express_1.default.json()); app.use(jobIngestionRoutes_1.default); app.use(jobSearchRoutes_1.default); +app.post('/api/ingest-jobs', jobIngestionController_1.ingestJobOffers); app.listen(PORT, () => { console.log(`Server is running on port ${PORT}`); }); diff --git a/backend/package-lock.json b/backend/package-lock.json index 14360c9..78f4f09 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -8,9 +8,6 @@ "name": "backend", "version": "1.0.0", "license": "ISC", - "dependencies": { - "typescript": "^5.8.3" - }, "devDependencies": { "@prisma/client": "^6.8.2", "@types/express": "^5.0.2", @@ -20,7 +17,9 @@ "express": "^5.1.0", "pg": "^8.16.0", "prisma": "^6.8.2", - "ts-node-dev": "^2.0.0" + "ts-node": "^10.9.2", + "ts-node-dev": "^2.0.0", + "typescript": "^5.8.3" } }, "node_modules/@cspotcode/source-map-support": { @@ -2080,6 +2079,7 @@ "version": "5.8.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.3.tgz", "integrity": "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==", + "dev": true, "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", diff --git a/backend/package.json b/backend/package.json index 415896c..2f8169c 100644 --- a/backend/package.json +++ b/backend/package.json @@ -3,17 +3,15 @@ "version": "1.0.0", "description": "", "main": "index.js", -"scripts": { - "start": "ts-node-dev --respawn --transpile-only src/index.ts", - "test": "echo \"Error: no test specified\" && exit 1" + "scripts": { + "start": "ts-node src/index.ts", + "build": "tsc", + "dev": "ts-node-dev src/index.ts" }, "keywords": [], "author": "", "license": "ISC", "type": "commonjs", - "dependencies": { - "typescript": "^5.8.3" - }, "devDependencies": { "@prisma/client": "^6.8.2", "@types/express": "^5.0.2", @@ -23,6 +21,8 @@ "express": "^5.1.0", "pg": "^8.16.0", "prisma": "^6.8.2", - "ts-node-dev": "^2.0.0" + "ts-node": "^10.9.2", + "ts-node-dev": "^2.0.0", + "typescript": "^5.8.3" } -} +} \ No newline at end of file diff --git a/backend/src/controllers/jobIngestionController.js b/backend/src/controllers/jobIngestionController.js deleted file mode 100644 index eeed9b9..0000000 --- a/backend/src/controllers/jobIngestionController.js +++ /dev/null @@ -1,55 +0,0 @@ -"use strict"; -var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { - function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } - return new (P || (P = Promise))(function (resolve, reject) { - function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } - function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } - function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } - step((generator = generator.apply(thisArg, _arguments || [])).next()); - }); -}; -var __importDefault = (this && this.__importDefault) || function (mod) { - return (mod && mod.__esModule) ? mod : { "default": mod }; -}; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.ingestJobOffers = void 0; -const client_1 = require("@prisma/client"); -const FranceTravailService_1 = __importDefault(require("../services/FranceTravailService")); -const prisma = new client_1.PrismaClient(); -const ingestJobOffers = (req, res) => __awaiter(void 0, void 0, void 0, function* () { - var _a, _b, _c, _d, _e; - try { - const jobOffers = yield FranceTravailService_1.default.getJobOffers({ range: '0-149' }); - for (const offre of jobOffers.resultats) { - const mappedOffer = { - id: offre.id, - title: offre.intitule, - description: offre.description, - publicationDate: new Date(offre.dateCreation), - romeCode: offre.romeCode, - romeLabel: offre.romeLibelle, - locationLabel: ((_a = offre.lieuTravail) === null || _a === void 0 ? void 0 : _a.libelle) || null, - postalCode: ((_b = offre.lieuTravail) === null || _b === void 0 ? void 0 : _b.codePostal) || null, - departmentCode: ((_c = offre.lieuTravail) === null || _c === void 0 ? void 0 : _c.codeDepartement) || null, - cityName: ((_d = offre.lieuTravail) === null || _d === void 0 ? void 0 : _d.ville) || null, - companyName: ((_e = offre.entreprise) === null || _e === void 0 ? void 0 : _e.nom) || null, - contractType: offre.typeContrat, - contractLabel: offre.libelleTypeContrat, - }; - yield prisma.jobOffer.upsert({ - where: { id: mappedOffer.id }, - update: mappedOffer, - create: mappedOffer, - }); - } - res.status(200).json({ - message: 'Job offers ingested successfully', - count: jobOffers.resultats.length, - }); - } - catch (error) { - console.error('Error ingesting job offers:', error); - res.status(500).json({ error: 'Failed to ingest job offers' }); - } -}); -exports.ingestJobOffers = ingestJobOffers; diff --git a/backend/src/controllers/jobIngestionController.ts b/backend/src/controllers/jobIngestionController.ts index 63b2fcc..58c1b9f 100644 --- a/backend/src/controllers/jobIngestionController.ts +++ b/backend/src/controllers/jobIngestionController.ts @@ -4,40 +4,89 @@ import FranceTravailService from '../services/FranceTravailService'; const prisma = new PrismaClient(); +// Fonction utilitaire pour introduire un délai +const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); + export const ingestJobOffers = async (req: Request, res: Response) => { + console.log('Ingesting job offers...'); // Log pour vérifier que le fichier TS est utilisé + + const BATCH_SIZE = 149; // Limite par requête, -1 pour une plage 0-149 + const TOTAL_OFFERS_TO_FETCH = 1000; // Nombre total d'offres que vous voulez essayer d'ingérer + const DELAY_MS = 150; // Délai en ms entre chaque requête pour respecter le rate limit (1000ms/10appels = 100ms, donc 150ms est prudent) + + let totalIngestedCount = 0; + let page = 0; + try { - const jobOffers = await FranceTravailService.getJobOffers({ range: '0-149' }); + for (let start = 0; start < TOTAL_OFFERS_TO_FETCH; start += (BATCH_SIZE + 1)) { + const end = start + BATCH_SIZE; + const range = `${start}-${end}`; + console.log(`Workspaceing offers with range: ${range}`); - for (const offre of jobOffers.resultats) { - const mappedOffer = { - id: offre.id, - title: offre.intitule, - description: offre.description, - publicationDate: new Date(offre.dateCreation), - romeCode: offre.romeCode, - romeLabel: offre.romeLibelle, - locationLabel: offre.lieuTravail?.libelle || null, - postalCode: offre.lieuTravail?.codePostal || null, - departmentCode: offre.lieuTravail?.codeDepartement || null, - cityName: offre.lieuTravail?.ville || null, - companyName: offre.entreprise?.nom || null, - contractType: offre.typeContrat, - contractLabel: offre.libelleTypeContrat, - }; + try { + const jobOffersResponse = await FranceTravailService.getJobOffers({ range }); + const jobOffers = jobOffersResponse.resultats; // Assurez-vous que la réponse contient bien un tableau 'resultats' - await prisma.jobOffer.upsert({ - where: { id: mappedOffer.id }, - update: mappedOffer, - create: mappedOffer, - }); + if (!jobOffers || jobOffers.length === 0) { + console.log(`No more offers found for range ${range}. Stopping ingestion.`); + break; // Arrête la boucle s'il n'y a plus d'offres + } + + for (const offre of jobOffers) { + const mappedOffer = { + id: String(offre.id), // S'assurer que l'ID est une string pour Prisma + title: offre.intitule || 'N/A', // Ajouter des valeurs par défaut pour les champs obligatoires si l'API peut retourner null + description: offre.description || 'N/A', + publicationDate: new Date(offre.dateCreation), + romeCode: offre.romeCode || null, + romeLabel: offre.romeLibelle || null, + locationLabel: offre.lieuTravail?.libelle || null, + postalCode: offre.lieuTravail?.codePostal || null, + departmentCode: offre.lieuTravail?.codeDepartement || null, + cityName: offre.lieuTravail?.ville || null, + companyName: offre.entreprise?.nom || null, + contractType: offre.typeContrat || null, + contractLabel: offre.libelleTypeContrat || null, + }; + + // Assurer la validité des dates pour Prisma + if (isNaN(mappedOffer.publicationDate.getTime())) { + console.warn(`Invalid dateCreation for offer ID: ${offre.id}. Using current date.`); + mappedOffer.publicationDate = new Date(); + } + + await prisma.jobOffer.upsert({ + where: { id: mappedOffer.id }, + update: mappedOffer, + create: mappedOffer, + }); + totalIngestedCount++; + } + + console.log(`Ingested ${jobOffers.length} offers for range ${range}. Total: ${totalIngestedCount}`); + + } catch (batchError) { + // Gérer les erreurs spécifiques à un appel de lot (ex: rate limit temporaire) + const axiosError = batchError as any; // Utiliser 'any' pour un accès flexible à la propriété 'response' + console.warn(`Error fetching batch for range ${range}:`, axiosError.response?.data || axiosError.message); + // Si c'est une erreur de rate limit, vous pourriez vouloir augmenter le délai ou réessayer + } + + // Introduire un délai avant la prochaine requête pour respecter le rate limit + if (start + (BATCH_SIZE + 1) < TOTAL_OFFERS_TO_FETCH) { + await sleep(DELAY_MS); + } } res.status(200).json({ - message: 'Job offers ingested successfully', - count: jobOffers.resultats.length, + message: `Job offers ingestion complete. Total offers processed: ${totalIngestedCount}`, + count: totalIngestedCount, }); + } catch (error) { - console.error('Error ingesting job offers:', error); + console.error('Fatal error during job offers ingestion process:', error); res.status(500).json({ error: 'Failed to ingest job offers' }); + } finally { + await prisma.$disconnect(); // Déconnecter Prisma après l'opération } -}; +}; \ No newline at end of file diff --git a/backend/src/controllers/jobSearchController.js b/backend/src/controllers/jobSearchController.js deleted file mode 100644 index e06c90b..0000000 --- a/backend/src/controllers/jobSearchController.js +++ /dev/null @@ -1,78 +0,0 @@ -"use strict"; -var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { - function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } - return new (P || (P = Promise))(function (resolve, reject) { - function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } - function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } - function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } - step((generator = generator.apply(thisArg, _arguments || [])).next()); - }); -}; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.searchLocalJobOffers = void 0; -const client_1 = require("@prisma/client"); -const prisma = new client_1.PrismaClient(); -const searchLocalJobOffers = (req, res) => __awaiter(void 0, void 0, void 0, function* () { - try { - const page = parseInt(req.query.page) || 1; - const limit = parseInt(req.query.limit) || 10; - const skip = (page - 1) * limit; - const take = limit; - const sortBy = req.query.sortBy || 'publicationDate'; - const sortOrder = req.query.sortOrder || 'desc'; - const keyword = req.query.keyword; - const location = req.query.location; - const contractType = req.query.contractType; - const where = {}; - if (keyword) { - where.OR = [ - { title: { contains: keyword, mode: 'insensitive' } }, - { description: { contains: keyword, mode: 'insensitive' } }, - ]; - } - if (location) { - where.AND = [ - ...(where.AND || []), - { - OR: [ - { locationLabel: { contains: location, mode: 'insensitive' } }, - { postalCode: { contains: location, mode: 'insensitive' } }, - { cityName: { contains: location, mode: 'insensitive' } }, - { departmentCode: { contains: location, mode: 'insensitive' } }, - ], - }, - ]; - } - if (contractType) { - where.AND = [ - ...(where.AND || []), - { contractType: contractType }, - ]; - } - const orderBy = {}; - if (sortBy) { - orderBy[sortBy] = sortOrder === 'asc' ? 'asc' : 'desc'; - } - else { - orderBy.publicationDate = 'desc'; // Tri par défaut - } - const jobs = yield prisma.jobOffer.findMany({ - skip, - take, - where, - orderBy, - }); - const total = yield prisma.jobOffer.count({ where }); - res.status(200).json({ - jobs, - total, - page, - limit, - }); - } - catch (error) { - console.error('Error searching job offers:', error); - res.status(500).json({ error: 'Failed to search job offers' }); - } -}); -exports.searchLocalJobOffers = searchLocalJobOffers; diff --git a/backend/src/index.ts b/backend/src/index.ts index c87771f..5584b9c 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -2,6 +2,7 @@ import express from 'express'; import dotenv from 'dotenv'; import jobIngestionRoutes from './routes/jobIngestionRoutes'; import jobSearchRoutes from './routes/jobSearchRoutes'; +import { ingestJobOffers } from './controllers/jobIngestionController'; dotenv.config(); @@ -12,7 +13,9 @@ app.use(express.json()); app.use(jobIngestionRoutes); app.use(jobSearchRoutes); +app.post('/api/ingest-jobs', ingestJobOffers); app.listen(PORT, () => { console.log(`Server is running on port ${PORT}`); -}); + +}); \ No newline at end of file