ab/jobia
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

augementation des ranges

Browse source
This commit is contained in:
el 2025-05-24 02:02:17 +02:00
parent 9653e55453
commit 7878062edc
9 changed files with 159 additions and 197 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -1,2 +1,3 @@
.env
/documentation/
node_modules

90
backend/dist/controllers/jobIngestionController.js vendored
View file

@ -7,39 +7,79 @@ exports.ingestJobOffers = void 0;
const client_1 = require("@prisma/client");
const FranceTravailService_1 = __importDefault(require("../services/FranceTravailService"));
const prisma = new client_1.PrismaClient();
// Fonction utilitaire pour introduire un délai
const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
const ingestJobOffers = async (req, res) => {
console.log('Ingesting job offers...'); // Log pour vérifier que le fichier TS est utilisé
const BATCH_SIZE = 149; // Limite par requête, -1 pour une plage 0-149
const TOTAL_OFFERS_TO_FETCH = 1000; // Nombre total d'offres que vous voulez essayer d'ingérer
const DELAY_MS = 150; // Délai en ms entre chaque requête pour respecter le rate limit (1000ms/10appels = 100ms, donc 150ms est prudent)
let totalIngestedCount = 0;
let page = 0;
try {
const jobOffers = await FranceTravailService_1.default.getJobOffers({ range: '0-149' });
for (const offre of jobOffers.resultats) {
const mappedOffer = {
id: offre.id,
title: offre.intitule,
description: offre.description,
publicationDate: new Date(offre.dateCreation),
romeCode: offre.romeCode,
romeLabel: offre.romeLibelle,
locationLabel: offre.lieuTravail?.libelle || null,
postalCode: offre.lieuTravail?.codePostal || null,
departmentCode: offre.lieuTravail?.codeDepartement || null,
cityName: offre.lieuTravail?.ville || null,
companyName: offre.entreprise?.nom || null,
contractType: offre.typeContrat,
contractLabel: offre.libelleTypeContrat,
};
await prisma.jobOffer.upsert({
where: { id: mappedOffer.id },
update: mappedOffer,
create: mappedOffer,
});
for (let start = 0; start < TOTAL_OFFERS_TO_FETCH; start += (BATCH_SIZE + 1)) {
const end = start + BATCH_SIZE;
const range = `${start}-${end}`;
console.log(`Workspaceing offers with range: ${range}`);
try {
const jobOffersResponse = await FranceTravailService_1.default.getJobOffers({ range });
const jobOffers = jobOffersResponse.resultats; // Assurez-vous que la réponse contient bien un tableau 'resultats'
if (!jobOffers || jobOffers.length === 0) {
console.log(`No more offers found for range ${range}. Stopping ingestion.`);
break; // Arrête la boucle s'il n'y a plus d'offres
}
for (const offre of jobOffers) {
const mappedOffer = {
id: String(offre.id), // S'assurer que l'ID est une string pour Prisma
title: offre.intitule || 'N/A', // Ajouter des valeurs par défaut pour les champs obligatoires si l'API peut retourner null
description: offre.description || 'N/A',
publicationDate: new Date(offre.dateCreation),
romeCode: offre.romeCode || null,
romeLabel: offre.romeLibelle || null,
locationLabel: offre.lieuTravail?.libelle || null,
postalCode: offre.lieuTravail?.codePostal || null,
departmentCode: offre.lieuTravail?.codeDepartement || null,
cityName: offre.lieuTravail?.ville || null,
companyName: offre.entreprise?.nom || null,
contractType: offre.typeContrat || null,
contractLabel: offre.libelleTypeContrat || null,
};
// Assurer la validité des dates pour Prisma
if (isNaN(mappedOffer.publicationDate.getTime())) {
console.warn(`Invalid dateCreation for offer ID: ${offre.id}. Using current date.`);
mappedOffer.publicationDate = new Date();
}
await prisma.jobOffer.upsert({
where: { id: mappedOffer.id },
update: mappedOffer,
create: mappedOffer,
});
totalIngestedCount++;
}
console.log(`Ingested ${jobOffers.length} offers for range ${range}. Total: ${totalIngestedCount}`);
}
catch (batchError) {
// Gérer les erreurs spécifiques à un appel de lot (ex: rate limit temporaire)
const axiosError = batchError; // Utiliser 'any' pour un accès flexible à la propriété 'response'
console.warn(`Error fetching batch for range ${range}:`, axiosError.response?.data || axiosError.message);
// Si c'est une erreur de rate limit, vous pourriez vouloir augmenter le délai ou réessayer
}
// Introduire un délai avant la prochaine requête pour respecter le rate limit
if (start + (BATCH_SIZE + 1) < TOTAL_OFFERS_TO_FETCH) {
await sleep(DELAY_MS);
}
}
res.status(200).json({
message: 'Job offers ingested successfully',
count: jobOffers.resultats.length,
message: `Job offers ingestion complete. Total offers processed: ${totalIngestedCount}`,
count: totalIngestedCount,
});
}
catch (error) {
console.error('Error ingesting job offers:', error);
console.error('Fatal error during job offers ingestion process:', error);
res.status(500).json({ error: 'Failed to ingest job offers' });
}
finally {
await prisma.$disconnect(); // Déconnecter Prisma après l'opération
}
};
exports.ingestJobOffers = ingestJobOffers;

2
backend/dist/index.js vendored
View file

@ -7,12 +7,14 @@ const express_1 = __importDefault(require("express"));
const dotenv_1 = __importDefault(require("dotenv"));
const jobIngestionRoutes_1 = __importDefault(require("./routes/jobIngestionRoutes"));
const jobSearchRoutes_1 = __importDefault(require("./routes/jobSearchRoutes"));
const jobIngestionController_1 = require("./controllers/jobIngestionController");
dotenv_1.default.config();
const app = (0, express_1.default)();
const PORT = process.env.PORT || 3000;
app.use(express_1.default.json());
app.use(jobIngestionRoutes_1.default);
app.use(jobSearchRoutes_1.default);
app.post('/api/ingest-jobs', jobIngestionController_1.ingestJobOffers);
app.listen(PORT, () => {
console.log(`Server is running on port ${PORT}`);
});

8
backend/package-lock.json generated
View file

@ -8,9 +8,6 @@
"name": "backend",
"version": "1.0.0",
"license": "ISC",
"dependencies": {
"typescript": "^5.8.3"
},
"devDependencies": {
"@prisma/client": "^6.8.2",
"@types/express": "^5.0.2",
@ -20,7 +17,9 @@
"express": "^5.1.0",
"pg": "^8.16.0",
"prisma": "^6.8.2",
"ts-node-dev": "^2.0.0"
"ts-node": "^10.9.2",
"ts-node-dev": "^2.0.0",
"typescript": "^5.8.3"
}
},
"node_modules/@cspotcode/source-map-support": {
@ -2080,6 +2079,7 @@
"version": "5.8.3",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.3.tgz",
"integrity": "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==",
"dev": true,
"license": "Apache-2.0",
"bin": {
"tsc": "bin/tsc",

16
backend/package.json
View file

@ -3,17 +3,15 @@
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {
"start": "ts-node-dev --respawn --transpile-only src/index.ts",
"test": "echo \"Error: no test specified\" && exit 1"
"scripts": {
"start": "ts-node src/index.ts",
"build": "tsc",
"dev": "ts-node-dev src/index.ts"
},
"keywords": [],
"author": "",
"license": "ISC",
"type": "commonjs",
"dependencies": {
"typescript": "^5.8.3"
},
"devDependencies": {
"@prisma/client": "^6.8.2",
"@types/express": "^5.0.2",
@ -23,6 +21,8 @@
"express": "^5.1.0",
"pg": "^8.16.0",
"prisma": "^6.8.2",
"ts-node-dev": "^2.0.0"
"ts-node": "^10.9.2",
"ts-node-dev": "^2.0.0",
"typescript": "^5.8.3"
}
}
}

55
backend/src/controllers/jobIngestionController.js
View file

@ -1,55 +0,0 @@
"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.ingestJobOffers = void 0;
const client_1 = require("@prisma/client");
const FranceTravailService_1 = __importDefault(require("../services/FranceTravailService"));
const prisma = new client_1.PrismaClient();
const ingestJobOffers = (req, res) => __awaiter(void 0, void 0, void 0, function* () {
var _a, _b, _c, _d, _e;
try {
const jobOffers = yield FranceTravailService_1.default.getJobOffers({ range: '0-149' });
for (const offre of jobOffers.resultats) {
const mappedOffer = {
id: offre.id,
title: offre.intitule,
description: offre.description,
publicationDate: new Date(offre.dateCreation),
romeCode: offre.romeCode,
romeLabel: offre.romeLibelle,
locationLabel: ((_a = offre.lieuTravail) === null || _a === void 0 ? void 0 : _a.libelle) || null,
postalCode: ((_b = offre.lieuTravail) === null || _b === void 0 ? void 0 : _b.codePostal) || null,
departmentCode: ((_c = offre.lieuTravail) === null || _c === void 0 ? void 0 : _c.codeDepartement) || null,
cityName: ((_d = offre.lieuTravail) === null || _d === void 0 ? void 0 : _d.ville) || null,
companyName: ((_e = offre.entreprise) === null || _e === void 0 ? void 0 : _e.nom) || null,
contractType: offre.typeContrat,
contractLabel: offre.libelleTypeContrat,
};
yield prisma.jobOffer.upsert({
where: { id: mappedOffer.id },
update: mappedOffer,
create: mappedOffer,
});
}
res.status(200).json({
message: 'Job offers ingested successfully',
count: jobOffers.resultats.length,
});
}
catch (error) {
console.error('Error ingesting job offers:', error);
res.status(500).json({ error: 'Failed to ingest job offers' });
}
});
exports.ingestJobOffers = ingestJobOffers;

101
backend/src/controllers/jobIngestionController.ts
View file

@ -4,40 +4,89 @@ import FranceTravailService from '../services/FranceTravailService';
const prisma = new PrismaClient();
// Fonction utilitaire pour introduire un délai
const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
export const ingestJobOffers = async (req: Request, res: Response) => {
console.log('Ingesting job offers...'); // Log pour vérifier que le fichier TS est utilisé
const BATCH_SIZE = 149; // Limite par requête, -1 pour une plage 0-149
const TOTAL_OFFERS_TO_FETCH = 1000; // Nombre total d'offres que vous voulez essayer d'ingérer
const DELAY_MS = 150; // Délai en ms entre chaque requête pour respecter le rate limit (1000ms/10appels = 100ms, donc 150ms est prudent)
let totalIngestedCount = 0;
let page = 0;
try {
const jobOffers = await FranceTravailService.getJobOffers({ range: '0-149' });
for (let start = 0; start < TOTAL_OFFERS_TO_FETCH; start += (BATCH_SIZE + 1)) {
const end = start + BATCH_SIZE;
const range = `${start}-${end}`;
console.log(`Workspaceing offers with range: ${range}`);
for (const offre of jobOffers.resultats) {
const mappedOffer = {
id: offre.id,
title: offre.intitule,
description: offre.description,
publicationDate: new Date(offre.dateCreation),
romeCode: offre.romeCode,
romeLabel: offre.romeLibelle,
locationLabel: offre.lieuTravail?.libelle || null,
postalCode: offre.lieuTravail?.codePostal || null,
departmentCode: offre.lieuTravail?.codeDepartement || null,
cityName: offre.lieuTravail?.ville || null,
companyName: offre.entreprise?.nom || null,
contractType: offre.typeContrat,
contractLabel: offre.libelleTypeContrat,
};
try {
const jobOffersResponse = await FranceTravailService.getJobOffers({ range });
const jobOffers = jobOffersResponse.resultats; // Assurez-vous que la réponse contient bien un tableau 'resultats'
await prisma.jobOffer.upsert({
where: { id: mappedOffer.id },
update: mappedOffer,
create: mappedOffer,
});
if (!jobOffers || jobOffers.length === 0) {
console.log(`No more offers found for range ${range}. Stopping ingestion.`);
break; // Arrête la boucle s'il n'y a plus d'offres
}
for (const offre of jobOffers) {
const mappedOffer = {
id: String(offre.id), // S'assurer que l'ID est une string pour Prisma
title: offre.intitule || 'N/A', // Ajouter des valeurs par défaut pour les champs obligatoires si l'API peut retourner null
description: offre.description || 'N/A',
publicationDate: new Date(offre.dateCreation),
romeCode: offre.romeCode || null,
romeLabel: offre.romeLibelle || null,
locationLabel: offre.lieuTravail?.libelle || null,
postalCode: offre.lieuTravail?.codePostal || null,
departmentCode: offre.lieuTravail?.codeDepartement || null,
cityName: offre.lieuTravail?.ville || null,
companyName: offre.entreprise?.nom || null,
contractType: offre.typeContrat || null,
contractLabel: offre.libelleTypeContrat || null,
};
// Assurer la validité des dates pour Prisma
if (isNaN(mappedOffer.publicationDate.getTime())) {
console.warn(`Invalid dateCreation for offer ID: ${offre.id}. Using current date.`);
mappedOffer.publicationDate = new Date();
}
await prisma.jobOffer.upsert({
where: { id: mappedOffer.id },
update: mappedOffer,
create: mappedOffer,
});
totalIngestedCount++;
}
console.log(`Ingested ${jobOffers.length} offers for range ${range}. Total: ${totalIngestedCount}`);
} catch (batchError) {
// Gérer les erreurs spécifiques à un appel de lot (ex: rate limit temporaire)
const axiosError = batchError as any; // Utiliser 'any' pour un accès flexible à la propriété 'response'
console.warn(`Error fetching batch for range ${range}:`, axiosError.response?.data || axiosError.message);
// Si c'est une erreur de rate limit, vous pourriez vouloir augmenter le délai ou réessayer
}
// Introduire un délai avant la prochaine requête pour respecter le rate limit
if (start + (BATCH_SIZE + 1) < TOTAL_OFFERS_TO_FETCH) {
await sleep(DELAY_MS);
}
}
res.status(200).json({
message: 'Job offers ingested successfully',
count: jobOffers.resultats.length,
message: `Job offers ingestion complete. Total offers processed: ${totalIngestedCount}`,
count: totalIngestedCount,
});
} catch (error) {
console.error('Error ingesting job offers:', error);
console.error('Fatal error during job offers ingestion process:', error);
res.status(500).json({ error: 'Failed to ingest job offers' });
} finally {
await prisma.$disconnect(); // Déconnecter Prisma après l'opération
}
};
};

78
backend/src/controllers/jobSearchController.js
View file

@ -1,78 +0,0 @@
"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.searchLocalJobOffers = void 0;
const client_1 = require("@prisma/client");
const prisma = new client_1.PrismaClient();
const searchLocalJobOffers = (req, res) => __awaiter(void 0, void 0, void 0, function* () {
try {
const page = parseInt(req.query.page) || 1;
const limit = parseInt(req.query.limit) || 10;
const skip = (page - 1) * limit;
const take = limit;
const sortBy = req.query.sortBy || 'publicationDate';
const sortOrder = req.query.sortOrder || 'desc';
const keyword = req.query.keyword;
const location = req.query.location;
const contractType = req.query.contractType;
const where = {};
if (keyword) {
where.OR = [
{ title: { contains: keyword, mode: 'insensitive' } },
{ description: { contains: keyword, mode: 'insensitive' } },
];
}
if (location) {
where.AND = [
...(where.AND || []),
{
OR: [
{ locationLabel: { contains: location, mode: 'insensitive' } },
{ postalCode: { contains: location, mode: 'insensitive' } },
{ cityName: { contains: location, mode: 'insensitive' } },
{ departmentCode: { contains: location, mode: 'insensitive' } },
],
},
];
}
if (contractType) {
where.AND = [
...(where.AND || []),
{ contractType: contractType },
];
}
const orderBy = {};
if (sortBy) {
orderBy[sortBy] = sortOrder === 'asc' ? 'asc' : 'desc';
}
else {
orderBy.publicationDate = 'desc'; // Tri par défaut
}
const jobs = yield prisma.jobOffer.findMany({
skip,
take,
where,
orderBy,
});
const total = yield prisma.jobOffer.count({ where });
res.status(200).json({
jobs,
total,
page,
limit,
});
}
catch (error) {
console.error('Error searching job offers:', error);
res.status(500).json({ error: 'Failed to search job offers' });
}
});
exports.searchLocalJobOffers = searchLocalJobOffers;

5
backend/src/index.ts
View file

@ -2,6 +2,7 @@ import express from 'express';
import dotenv from 'dotenv';
import jobIngestionRoutes from './routes/jobIngestionRoutes';
import jobSearchRoutes from './routes/jobSearchRoutes';
import { ingestJobOffers } from './controllers/jobIngestionController';
dotenv.config();
@ -12,7 +13,9 @@ app.use(express.json());
app.use(jobIngestionRoutes);
app.use(jobSearchRoutes);
app.post('/api/ingest-jobs', ingestJobOffers);
app.listen(PORT, () => {
console.log(`Server is running on port ${PORT}`);
});
});