augementation des ranges

.gitignore

@@ -1,2 +1,3 @@
 .env
 /documentation/
+node_modules

backend/dist/controllers/jobIngestionController.js

@@ -7,39 +7,79 @@ exports.ingestJobOffers = void 0;
 const client_1 = require("@prisma/client");
 const FranceTravailService_1 = __importDefault(require("../services/FranceTravailService"));
 const prisma = new client_1.PrismaClient();
+// Fonction utilitaire pour introduire un délai
+const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
 const ingestJobOffers = async (req, res) => {
+    console.log('Ingesting job offers...'); // Log pour vérifier que le fichier TS est utilisé
+    const BATCH_SIZE = 149; // Limite par requête, -1 pour une plage 0-149
+    const TOTAL_OFFERS_TO_FETCH = 1000; // Nombre total d'offres que vous voulez essayer d'ingérer
+    const DELAY_MS = 150; // Délai en ms entre chaque requête pour respecter le rate limit (1000ms/10appels = 100ms, donc 150ms est prudent)
+    let totalIngestedCount = 0;
+    let page = 0;
     try {
-        const jobOffers = await FranceTravailService_1.default.getJobOffers({ range: '0-149' });
+        for (let start = 0; start < TOTAL_OFFERS_TO_FETCH; start += (BATCH_SIZE + 1)) {
-        for (const offre of jobOffers.resultats) {
+            const end = start + BATCH_SIZE;
+            const range = `${start}-${end}`;
+            console.log(`Workspaceing offers with range: ${range}`);
+            try {
+                const jobOffersResponse = await FranceTravailService_1.default.getJobOffers({ range });
+                const jobOffers = jobOffersResponse.resultats; // Assurez-vous que la réponse contient bien un tableau 'resultats'
+                if (!jobOffers || jobOffers.length === 0) {
+                    console.log(`No more offers found for range ${range}. Stopping ingestion.`);
+                    break; // Arrête la boucle s'il n'y a plus d'offres
+                }
+                for (const offre of jobOffers) {
                     const mappedOffer = {
-                id: offre.id,
+                        id: String(offre.id), // S'assurer que l'ID est une string pour Prisma
-                title: offre.intitule,
+                        title: offre.intitule || 'N/A', // Ajouter des valeurs par défaut pour les champs obligatoires si l'API peut retourner null
-                description: offre.description,
+                        description: offre.description || 'N/A',
                         publicationDate: new Date(offre.dateCreation),
-                romeCode: offre.romeCode,
+                        romeCode: offre.romeCode || null,
-                romeLabel: offre.romeLibelle,
+                        romeLabel: offre.romeLibelle || null,
                         locationLabel: offre.lieuTravail?.libelle || null,
                         postalCode: offre.lieuTravail?.codePostal || null,
                         departmentCode: offre.lieuTravail?.codeDepartement || null,
                         cityName: offre.lieuTravail?.ville || null,
                         companyName: offre.entreprise?.nom || null,
-                contractType: offre.typeContrat,
+                        contractType: offre.typeContrat || null,
-                contractLabel: offre.libelleTypeContrat,
+                        contractLabel: offre.libelleTypeContrat || null,
                     };
+                    // Assurer la validité des dates pour Prisma
+                    if (isNaN(mappedOffer.publicationDate.getTime())) {
+                        console.warn(`Invalid dateCreation for offer ID: ${offre.id}. Using current date.`);
+                        mappedOffer.publicationDate = new Date();
+                    }
                     await prisma.jobOffer.upsert({
                         where: { id: mappedOffer.id },
                         update: mappedOffer,
                         create: mappedOffer,
                     });
+                    totalIngestedCount++;
+                }
+                console.log(`Ingested ${jobOffers.length} offers for range ${range}. Total: ${totalIngestedCount}`);
+            }
+            catch (batchError) {
+                // Gérer les erreurs spécifiques à un appel de lot (ex: rate limit temporaire)
+                const axiosError = batchError; // Utiliser 'any' pour un accès flexible à la propriété 'response'
+                console.warn(`Error fetching batch for range ${range}:`, axiosError.response?.data || axiosError.message);
+                // Si c'est une erreur de rate limit, vous pourriez vouloir augmenter le délai ou réessayer
+            }
+            // Introduire un délai avant la prochaine requête pour respecter le rate limit
+            if (start + (BATCH_SIZE + 1) < TOTAL_OFFERS_TO_FETCH) {
+                await sleep(DELAY_MS);
+            }
         }
         res.status(200).json({
-            message: 'Job offers ingested successfully',
+            message: `Job offers ingestion complete. Total offers processed: ${totalIngestedCount}`,
-            count: jobOffers.resultats.length,
+            count: totalIngestedCount,
         });
     }
     catch (error) {
-        console.error('Error ingesting job offers:', error);
+        console.error('Fatal error during job offers ingestion process:', error);
         res.status(500).json({ error: 'Failed to ingest job offers' });
     }
+    finally {
+        await prisma.$disconnect(); // Déconnecter Prisma après l'opération
+    }
 };
 exports.ingestJobOffers = ingestJobOffers;

backend/dist/index.js

@@ -7,12 +7,14 @@ const express_1 = __importDefault(require("express"));
 const dotenv_1 = __importDefault(require("dotenv"));
 const jobIngestionRoutes_1 = __importDefault(require("./routes/jobIngestionRoutes"));
 const jobSearchRoutes_1 = __importDefault(require("./routes/jobSearchRoutes"));
+const jobIngestionController_1 = require("./controllers/jobIngestionController");
 dotenv_1.default.config();
 const app = (0, express_1.default)();
 const PORT = process.env.PORT || 3000;
 app.use(express_1.default.json());
 app.use(jobIngestionRoutes_1.default);
 app.use(jobSearchRoutes_1.default);
+app.post('/api/ingest-jobs', jobIngestionController_1.ingestJobOffers);
 app.listen(PORT, () => {
     console.log(`Server is running on port ${PORT}`);
 });

backend/package-lock.json

@@ -8,9 +8,6 @@
       "name": "backend",
       "version": "1.0.0",
       "license": "ISC",
-      "dependencies": {
-        "typescript": "^5.8.3"
-      },
       "devDependencies": {
         "@prisma/client": "^6.8.2",
         "@types/express": "^5.0.2",
@@ -20,7 +17,9 @@
         "express": "^5.1.0",
         "pg": "^8.16.0",
         "prisma": "^6.8.2",
-        "ts-node-dev": "^2.0.0"
+        "ts-node": "^10.9.2",
+        "ts-node-dev": "^2.0.0",
+        "typescript": "^5.8.3"
       }
     },
     "node_modules/@cspotcode/source-map-support": {
@@ -2080,6 +2079,7 @@
       "version": "5.8.3",
       "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.3.tgz",
       "integrity": "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==",
+      "dev": true,
       "license": "Apache-2.0",
       "bin": {
         "tsc": "bin/tsc",

backend/package.json

@@ -3,17 +3,15 @@
   "version": "1.0.0",
   "description": "",
   "main": "index.js",
-"scripts": {
+  "scripts": {
-    "start": "ts-node-dev --respawn --transpile-only src/index.ts",
+    "start": "ts-node src/index.ts",
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "build": "tsc",
+    "dev": "ts-node-dev src/index.ts"
   },
   "keywords": [],
   "author": "",
   "license": "ISC",
   "type": "commonjs",
-  "dependencies": {
-    "typescript": "^5.8.3"
-  },
   "devDependencies": {
     "@prisma/client": "^6.8.2",
     "@types/express": "^5.0.2",
@@ -23,6 +21,8 @@
     "express": "^5.1.0",
     "pg": "^8.16.0",
     "prisma": "^6.8.2",
-    "ts-node-dev": "^2.0.0"
+    "ts-node": "^10.9.2",
+    "ts-node-dev": "^2.0.0",
+    "typescript": "^5.8.3"
   }
 }

backend/src/controllers/jobIngestionController.js

@@ -1,55 +0,0 @@
-"use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ingestJobOffers = void 0;
-const client_1 = require("@prisma/client");
-const FranceTravailService_1 = __importDefault(require("../services/FranceTravailService"));
-const prisma = new client_1.PrismaClient();
-const ingestJobOffers = (req, res) => __awaiter(void 0, void 0, void 0, function* () {
-    var _a, _b, _c, _d, _e;
-    try {
-        const jobOffers = yield FranceTravailService_1.default.getJobOffers({ range: '0-149' });
-        for (const offre of jobOffers.resultats) {
-            const mappedOffer = {
-                id: offre.id,
-                title: offre.intitule,
-                description: offre.description,
-                publicationDate: new Date(offre.dateCreation),
-                romeCode: offre.romeCode,
-                romeLabel: offre.romeLibelle,
-                locationLabel: ((_a = offre.lieuTravail) === null || _a === void 0 ? void 0 : _a.libelle) || null,
-                postalCode: ((_b = offre.lieuTravail) === null || _b === void 0 ? void 0 : _b.codePostal) || null,
-                departmentCode: ((_c = offre.lieuTravail) === null || _c === void 0 ? void 0 : _c.codeDepartement) || null,
-                cityName: ((_d = offre.lieuTravail) === null || _d === void 0 ? void 0 : _d.ville) || null,
-                companyName: ((_e = offre.entreprise) === null || _e === void 0 ? void 0 : _e.nom) || null,
-                contractType: offre.typeContrat,
-                contractLabel: offre.libelleTypeContrat,
-            };
-            yield prisma.jobOffer.upsert({
-                where: { id: mappedOffer.id },
-                update: mappedOffer,
-                create: mappedOffer,
-            });
-        }
-        res.status(200).json({
-            message: 'Job offers ingested successfully',
-            count: jobOffers.resultats.length,
-        });
-    }
-    catch (error) {
-        console.error('Error ingesting job offers:', error);
-        res.status(500).json({ error: 'Failed to ingest job offers' });
-    }
-});
-exports.ingestJobOffers = ingestJobOffers;

backend/src/controllers/jobIngestionController.ts

@@ -4,40 +4,89 @@ import FranceTravailService from '../services/FranceTravailService';
 
 const prisma = new PrismaClient();
 
-export const ingestJobOffers = async (req: Request, res: Response) => {
+// Fonction utilitaire pour introduire un délai
-  try {
+const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
-    const jobOffers = await FranceTravailService.getJobOffers({ range: '0-149' });
 
-    for (const offre of jobOffers.resultats) {
+export const ingestJobOffers = async (req: Request, res: Response) => {
+  console.log('Ingesting job offers...'); // Log pour vérifier que le fichier TS est utilisé
+
+  const BATCH_SIZE = 149; // Limite par requête, -1 pour une plage 0-149
+  const TOTAL_OFFERS_TO_FETCH = 1000; // Nombre total d'offres que vous voulez essayer d'ingérer
+  const DELAY_MS = 150; // Délai en ms entre chaque requête pour respecter le rate limit (1000ms/10appels = 100ms, donc 150ms est prudent)
+
+  let totalIngestedCount = 0;
+  let page = 0;
+
+  try {
+    for (let start = 0; start < TOTAL_OFFERS_TO_FETCH; start += (BATCH_SIZE + 1)) {
+      const end = start + BATCH_SIZE;
+      const range = `${start}-${end}`;
+      console.log(`Workspaceing offers with range: ${range}`);
+
+      try {
+        const jobOffersResponse = await FranceTravailService.getJobOffers({ range });
+        const jobOffers = jobOffersResponse.resultats; // Assurez-vous que la réponse contient bien un tableau 'resultats'
+
+        if (!jobOffers || jobOffers.length === 0) {
+          console.log(`No more offers found for range ${range}. Stopping ingestion.`);
+          break; // Arrête la boucle s'il n'y a plus d'offres
+        }
+
+        for (const offre of jobOffers) {
           const mappedOffer = {
-        id: offre.id,
+            id: String(offre.id), // S'assurer que l'ID est une string pour Prisma
-        title: offre.intitule,
+            title: offre.intitule || 'N/A', // Ajouter des valeurs par défaut pour les champs obligatoires si l'API peut retourner null
-        description: offre.description,
+            description: offre.description || 'N/A',
             publicationDate: new Date(offre.dateCreation),
-        romeCode: offre.romeCode,
+            romeCode: offre.romeCode || null,
-        romeLabel: offre.romeLibelle,
+            romeLabel: offre.romeLibelle || null,
             locationLabel: offre.lieuTravail?.libelle || null,
             postalCode: offre.lieuTravail?.codePostal || null,
             departmentCode: offre.lieuTravail?.codeDepartement || null,
             cityName: offre.lieuTravail?.ville || null,
             companyName: offre.entreprise?.nom || null,
-        contractType: offre.typeContrat,
+            contractType: offre.typeContrat || null,
-        contractLabel: offre.libelleTypeContrat,
+            contractLabel: offre.libelleTypeContrat || null,
           };
 
+          // Assurer la validité des dates pour Prisma
+          if (isNaN(mappedOffer.publicationDate.getTime())) {
+            console.warn(`Invalid dateCreation for offer ID: ${offre.id}. Using current date.`);
+            mappedOffer.publicationDate = new Date();
+          }
+
           await prisma.jobOffer.upsert({
             where: { id: mappedOffer.id },
             update: mappedOffer,
             create: mappedOffer,
           });
+          totalIngestedCount++;
+        }
+
+        console.log(`Ingested ${jobOffers.length} offers for range ${range}. Total: ${totalIngestedCount}`);
+
+      } catch (batchError) {
+        // Gérer les erreurs spécifiques à un appel de lot (ex: rate limit temporaire)
+        const axiosError = batchError as any; // Utiliser 'any' pour un accès flexible à la propriété 'response'
+        console.warn(`Error fetching batch for range ${range}:`, axiosError.response?.data || axiosError.message);
+        // Si c'est une erreur de rate limit, vous pourriez vouloir augmenter le délai ou réessayer
+      }
+
+      // Introduire un délai avant la prochaine requête pour respecter le rate limit
+      if (start + (BATCH_SIZE + 1) < TOTAL_OFFERS_TO_FETCH) {
+        await sleep(DELAY_MS);
+      }
     }
 
     res.status(200).json({
-      message: 'Job offers ingested successfully',
+      message: `Job offers ingestion complete. Total offers processed: ${totalIngestedCount}`,
-      count: jobOffers.resultats.length,
+      count: totalIngestedCount,
     });
+
   } catch (error) {
-    console.error('Error ingesting job offers:', error);
+    console.error('Fatal error during job offers ingestion process:', error);
     res.status(500).json({ error: 'Failed to ingest job offers' });
+  } finally {
+    await prisma.$disconnect(); // Déconnecter Prisma après l'opération
   }
 };

backend/src/controllers/jobSearchController.js

@@ -1,78 +0,0 @@
-"use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.searchLocalJobOffers = void 0;
-const client_1 = require("@prisma/client");
-const prisma = new client_1.PrismaClient();
-const searchLocalJobOffers = (req, res) => __awaiter(void 0, void 0, void 0, function* () {
-    try {
-        const page = parseInt(req.query.page) || 1;
-        const limit = parseInt(req.query.limit) || 10;
-        const skip = (page - 1) * limit;
-        const take = limit;
-        const sortBy = req.query.sortBy || 'publicationDate';
-        const sortOrder = req.query.sortOrder || 'desc';
-        const keyword = req.query.keyword;
-        const location = req.query.location;
-        const contractType = req.query.contractType;
-        const where = {};
-        if (keyword) {
-            where.OR = [
-                { title: { contains: keyword, mode: 'insensitive' } },
-                { description: { contains: keyword, mode: 'insensitive' } },
-            ];
-        }
-        if (location) {
-            where.AND = [
-                ...(where.AND || []),
-                {
-                    OR: [
-                        { locationLabel: { contains: location, mode: 'insensitive' } },
-                        { postalCode: { contains: location, mode: 'insensitive' } },
-                        { cityName: { contains: location, mode: 'insensitive' } },
-                        { departmentCode: { contains: location, mode: 'insensitive' } },
-                    ],
-                },
-            ];
-        }
-        if (contractType) {
-            where.AND = [
-                ...(where.AND || []),
-                { contractType: contractType },
-            ];
-        }
-        const orderBy = {};
-        if (sortBy) {
-            orderBy[sortBy] = sortOrder === 'asc' ? 'asc' : 'desc';
-        }
-        else {
-            orderBy.publicationDate = 'desc'; // Tri par défaut
-        }
-        const jobs = yield prisma.jobOffer.findMany({
-            skip,
-            take,
-            where,
-            orderBy,
-        });
-        const total = yield prisma.jobOffer.count({ where });
-        res.status(200).json({
-            jobs,
-            total,
-            page,
-            limit,
-        });
-    }
-    catch (error) {
-        console.error('Error searching job offers:', error);
-        res.status(500).json({ error: 'Failed to search job offers' });
-    }
-});
-exports.searchLocalJobOffers = searchLocalJobOffers;

backend/src/index.ts

@@ -2,6 +2,7 @@ import express from 'express';
 import dotenv from 'dotenv';
 import jobIngestionRoutes from './routes/jobIngestionRoutes';
 import jobSearchRoutes from './routes/jobSearchRoutes';
+import { ingestJobOffers } from './controllers/jobIngestionController';
 
 dotenv.config();
 
@@ -12,7 +13,9 @@ app.use(express.json());
 
 app.use(jobIngestionRoutes);
 app.use(jobSearchRoutes);
+app.post('/api/ingest-jobs', ingestJobOffers);
 
 app.listen(PORT, () => {
   console.log(`Server is running on port ${PORT}`);
+  
 });