import { AIProcessor } from "@helpers/ai_processor";
import { Chat } from "@models/Chat.model";
import { Investigation } from "@models/investigation/investigation.model";
import { Metadata } from "@models/Metadata.model";
import { Prompt } from "@models/Prompt.model";
import { InvestigationStatus } from "@typez/investigation";
import { getLogger } from "@utils/asyncLocalStorage";
import { subjectsFactory } from "@utils/subjectsFactory";
import yamlModule from "js-yaml";
import { Types } from "mongoose";
import { InvestigationBuilder } from "./investigation.builder";
import { convertInvestigationFromModel } from "../helpers/investigation";
import { AssistantInvestigationFormat } from "../schemas/assistant.validation";
const MAX_INVESTIGATION_VERSIONS = 20;
/**
* Builds MongoDB aggregation stage for relevance scoring with priorities
* Priority order: Title (highest) > Curriculum/Unit/Lesson/Grade > Objectives/Goals/AnalyticalFacts/NGSS > Day/Steps/Objects
* Exact matches score higher than partial matches
* @category Services
* @param {string} escapedSearchTerm - Escaped search term for regex matching
* @returns {object} MongoDB $addFields stage with relevanceScore calculation
*/
function buildRelevanceScoreStage(escapedSearchTerm) {
const searchTermLower = escapedSearchTerm.toLowerCase();
const exactMatchRegex = `^${searchTermLower}$`;
const partialMatchRegex = searchTermLower;
// Helper to safely convert field value to string (handles arrays, nulls, etc.)
// Inline function that returns the expression object for a given field path
const safeToStringExpr = (fieldPath) => ({
$cond: {
if: { $isArray: fieldPath },
then: {
$ifNull: [{ $toString: { $arrayElemAt: [fieldPath, 0] } }, ""],
},
else: {
$ifNull: [
{
$convert: {
input: fieldPath,
to: "string",
onError: "",
onNull: "",
},
},
"",
],
},
},
});
// Helper to create a regex match expression for a field path
const createRegexMatch = (fieldPath, regexPattern) => ({
$regexMatch: {
input: {
$toLower: safeToStringExpr(fieldPath),
},
regex: regexPattern,
},
});
// Helper to check if any element in an array field matches the regex
const arrayMatchExists = (arrayFieldPath, elementAlias, elementFieldPath, regexPattern) => ({
$gt: [
{
$size: {
$filter: {
input: {
$cond: {
if: { $isArray: arrayFieldPath },
then: arrayFieldPath,
else: [],
},
},
as: elementAlias,
cond: createRegexMatch(elementFieldPath, regexPattern),
},
},
},
0,
],
});
const scoreExpressions = [
// Title exact match (highest priority: 1000 points)
{
$cond: [createRegexMatch("$title.value", exactMatchRegex), 1000, 0],
},
// Title partial match (high priority: 500 points)
{
$cond: [createRegexMatch("$title.value", partialMatchRegex), 500, 0],
},
// High-priority fields exact match (curriculum, unit, lesson, grade: 300 points)
{
$cond: [
{
$or: [
createRegexMatch("$curriculum.value", exactMatchRegex),
createRegexMatch("$unitNumberAndTitle.value", exactMatchRegex),
createRegexMatch("$lessonNumberAndTitle.value", exactMatchRegex),
createRegexMatch("$grade.value", exactMatchRegex),
],
},
300,
0,
],
},
// High-priority fields partial match (curriculum, unit, lesson, grade: 150 points)
{
$cond: [
{
$or: [
createRegexMatch("$curriculum.value", partialMatchRegex),
createRegexMatch("$unitNumberAndTitle.value", partialMatchRegex),
createRegexMatch("$lessonNumberAndTitle.value", partialMatchRegex),
createRegexMatch("$grade.value", partialMatchRegex),
],
},
150,
0,
],
},
// Medium-priority fields exact match (objectives, goals, analyticalFacts, ngss: 100 points)
{
$cond: [
{
$or: [
createRegexMatch("$objectives.value", exactMatchRegex),
createRegexMatch("$goals.value", exactMatchRegex),
createRegexMatch("$analyticalFacts.value", exactMatchRegex),
createRegexMatch("$ngss.value", exactMatchRegex),
],
},
100,
0,
],
},
// Medium-priority fields partial match (objectives, goals, analyticalFacts, ngss: 50 points)
{
$cond: [
{
$or: [
createRegexMatch("$objectives.value", partialMatchRegex),
createRegexMatch("$goals.value", partialMatchRegex),
createRegexMatch("$analyticalFacts.value", partialMatchRegex),
createRegexMatch("$ngss.value", partialMatchRegex),
],
},
50,
0,
],
},
// Lower-priority fields: Day exact match (50 points)
{
$cond: [createRegexMatch("$day.value", exactMatchRegex), 50, 0],
},
// Lower-priority fields: Day partial match (25 points)
{
$cond: [createRegexMatch("$day.value", partialMatchRegex), 25, 0],
},
// Steps: title exact match (50 points)
{
$cond: [arrayMatchExists("$steps", "step", "$$step.title.value", exactMatchRegex), 50, 0],
},
// Steps: title partial match (25 points)
{
$cond: [arrayMatchExists("$steps", "step", "$$step.title.value", partialMatchRegex), 25, 0],
},
// Steps: descriptionEn exact match (50 points)
{
$cond: [
arrayMatchExists("$steps", "step", "$$step.descriptionEn.value", exactMatchRegex),
50,
0,
],
},
// Steps: descriptionEn partial match (25 points)
{
$cond: [
arrayMatchExists("$steps", "step", "$$step.descriptionEn.value", partialMatchRegex),
25,
0,
],
},
// Objects: name exact match (50 points)
{
$cond: [arrayMatchExists("$objects", "obj", "$$obj.name.value", exactMatchRegex), 50, 0],
},
// Objects: name partial match (25 points)
{
$cond: [arrayMatchExists("$objects", "obj", "$$obj.name.value", partialMatchRegex), 25, 0],
},
];
return {
$addFields: {
relevanceScore: {
$add: scoreExpressions,
},
},
};
}
/**
* Investigation Service
* @category Services
*/
export class InvestigationService {
aiProcessor;
/**
* Constructor
* @category Services
*/
constructor() {
this.aiProcessor = new AIProcessor();
}
/**
* Recursively sets all `aiGeneratedValue` fields to `null`.
*
* This method is schema-agnostic and works regardless of changes to the investigation structure.
* @category Services
* @param {Record<string, unknown>} obj - The object whose `aiGeneratedValue` fields should be cleared.
*/
removeAiGeneratedValues(obj) {
if (!obj || typeof obj !== "object") {
return;
}
// Set aiGeneratedValue to null if it exists
if ("aiGeneratedValue" in obj) {
obj.aiGeneratedValue = null;
}
// Recursively process all properties
for (const key in obj) {
if (obj.hasOwnProperty(key)) {
const value = obj[key];
// If it's an object, recurse into it
if (value && typeof value === "object" && !Array.isArray(value)) {
this.removeAiGeneratedValues(value);
}
// If it's an array, process each element
else if (Array.isArray(value)) {
value.forEach((item) => {
if (item && typeof item === "object") {
this.removeAiGeneratedValues(item);
}
});
}
}
}
}
/**
* Creates a new investigation.
* @category Services
* @param {ICreateInvestigationDto} createDto - The data required to create the investigation.
* @returns {Promise<IInvestigation>} - The created investigation.
*/
async createInvestigation(createDto) {
const logger = getLogger();
try {
logger.info({ createDto }, "Creating new investigation");
// Use builder to create clean investigation data (default: human-created flow)
const investigationData = InvestigationBuilder.build(createDto, false);
logger.info({ investigationData }, "Investigation data by builder is built !");
// Create the investigation document
const investigation = new Investigation(investigationData);
logger.info({ investigation }, "Investigation document created ! ");
// Save the investigation
const savedInvestigation = await investigation.save();
logger.info({ investigationId: savedInvestigation._id }, "Investigation created successfully");
// Convert to response DTO
// TODO convert this to use zod
// const responseDto = {
// id: savedInvestigation._id.toString(),
// status: savedInvestigation.status,
// isLocked: savedInvestigation.isLocked,
// lockedBy: savedInvestigation.lockedBy?.toString() || null,
// lockedAt: savedInvestigation.lockedAt,
// originalRequest: savedInvestigation?.originalRequest?.value || "",
// curriculum: savedInvestigation?.curriculum?.value || "",
// gradeAndUnit: savedInvestigation?.gradeAndUnit?.value || "",
// title: savedInvestigation?.title?.value || "",
// unitTitle: savedInvestigation?.unitTitle?.value || "",
// objectives: savedInvestigation?.objectives?.value || "",
// discussionTopics: savedInvestigation?.discussionTopics?.value || "",
// trivia: savedInvestigation?.trivia?.value || "",
// analyticalFacts: savedInvestigation?.analyticalFacts?.value || "",
// introAndGoals: savedInvestigation?.introAndGoals?.value || "",
// steps: convertStepsToSimple(savedInvestigation?.steps || []),
// objects: savedInvestigation?.objects || [],
// observers: savedInvestigation?.observers,
// calibrationVariability: savedInvestigation?.calibrationVariability?.value || null,
// measurementVariability: savedInvestigation?.measurementVariability?.value || null,
// outcomeVariability: savedInvestigation?.outcomeVariability?.value || null,
// correspondence: savedInvestigation?.correspondence?.value || "",
// differences: savedInvestigation?.differences?.value || "",
// references: savedInvestigation?.references?.value || "",
// metadata: {
// dateOfDevelopment: null,
// dateOfDevelopmentDelivery: null,
// dateOfPublishing: savedInvestigation?.metadata?.dateOfPublishing,
// author: savedInvestigation?.metadata?.author?.toString() || null,
// editors: savedInvestigation?.metadata?.editors?.map((id) => id.toString()) || [],
// views: savedInvestigation?.metadata?.views,
// sourceInvestigation:
// savedInvestigation?.metadata?.sourceInvestigation?.toString() || null,
// dateOfCreationClone: savedInvestigation?.metadata?.dateOfCreationClone,
// authorClone: savedInvestigation?.metadata?.authorClone?.toString() || null,
// sourceInvestigationClone:
// savedInvestigation?.metadata?.sourceInvestigationClone?.toString() || null,
// },
// createdAt: savedInvestigation.createdAt,
// updatedAt: savedInvestigation.updatedAt,
// };
return savedInvestigation;
}
catch (error) {
logger.error({ error, createDto }, "Failed to create investigation");
throw new Error("Failed to create investigation");
}
}
/**
* Retrieves a single investigation by its ID.
* @category Services
* @param {string} investigationId - The ID of the investigation to retrieve.
* @returns {Promise<IInvestigation>} - The investigation data.
* @throws {Error} If the investigation is not found.
*/
async getInvestigationById(investigationId) {
const logger = getLogger();
try {
logger.info({ investigationId }, "Fetching investigation by ID");
const investigation = await Investigation.findById(investigationId);
if (!investigation) {
throw new Error("Investigation not found");
}
// Get the version at currentVersionIndex
// Note: Index 0 = versions[0] (oldest saved version)
// Index 1 = versions[1]
// ...
// Index versions.length - 1 = versions[versions.length - 1] (newest saved version)
// Index versions.length = current state (not in versions array)
const currentIndex = investigation.currentVersionIndex ?? 0;
const versions = Array.isArray(investigation.versions) ? investigation.versions : [];
// Versions are stored with newest at the end of the array (no need to sort)
// versions[0] = oldest saved version
// versions[versions.length - 1] = newest saved version
const sortedVersions = versions;
// Index versions.length means current state (not in versions array)
if (currentIndex === sortedVersions.length) {
logger.info({ investigationId, currentIndex }, "Returning current investigation (index = versions.length = current state)");
return investigation;
}
// If currentIndex is out of bounds, return current investigation
// Valid indices: 0 to versions.length (inclusive)
if (currentIndex < 0 || currentIndex > sortedVersions.length) {
logger.info({ investigationId, currentIndex, versionsLength: sortedVersions.length }, "Returning current investigation (invalid index)");
return investigation;
}
// Get the version at currentIndex (index 0 = versions[0], index 1 = versions[1], etc.)
const versionToRestore = sortedVersions[currentIndex];
if (!versionToRestore || !versionToRestore.snapshot) {
logger.info({ investigationId, currentIndex, versionsLength: sortedVersions.length }, "Version snapshot not found, returning current investigation");
return investigation;
}
logger.info({
investigationId,
currentIndex,
versionNumber: versionToRestore.versionNumber,
versionsLength: sortedVersions.length,
availableVersions: sortedVersions.map((v) => v.versionNumber),
}, "Restoring investigation from version snapshot");
// Restore the investigation from the snapshot
const restoredInvestigation = this.restoreInvestigationFromSnapshot(investigation, versionToRestore.snapshot);
logger.info({ investigationId, currentIndex, versionNumber: versionToRestore.versionNumber }, "Returning investigation at version index");
return restoredInvestigation;
}
catch (error) {
logger.error({ error, investigationId }, "Failed to fetch investigation");
throw error;
}
}
/**
* Restores an investigation from a snapshot
* @private
* @param {IInvestigation} investigation - The investigation document to restore
* @param {IInvestigationSnapshot} snapshot - The snapshot data to apply
* @returns {IInvestigation} - The investigation with snapshot data applied
*/
restoreInvestigationFromSnapshot(investigation, snapshot) {
// Apply snapshot data directly to the investigation document
// This preserves the Mongoose document structure
const investigationRecord = investigation;
Object.keys(snapshot).forEach((key) => {
if (key !== "_id" &&
key !== "id" &&
key !== "__v" &&
key !== "versions" &&
key !== "currentVersionIndex" &&
key !== "createdAt" &&
key !== "updatedAt") {
investigationRecord[key] = snapshot[key];
}
});
return investigation;
}
trackFieldHistory(oldField, newField, fieldPath, updatedBy, historyUpdates) {
if (oldField && typeof oldField === "object" && "value" in oldField) {
const oldValue = oldField.value;
let newValue = null;
if (newField && typeof newField === "object" && "value" in newField) {
newValue = newField.value ?? null;
}
else if (newField !== undefined) {
newValue = newField;
}
if (oldValue !== newValue && oldValue !== undefined && oldValue !== null) {
historyUpdates[fieldPath] = {
historyEntry: {
value: oldValue,
updatedBy,
updatedAt: new Date(),
},
fieldPath,
};
}
}
}
saveFieldHistory(investigation, updateFields, updatedBy) {
const logger = getLogger();
const historyUpdates = {};
for (const [fieldPath, newValue] of Object.entries(updateFields)) {
if (fieldPath === "metadata.dateModified") {
continue;
}
if (fieldPath === "objects") {
const oldObjects = investigation.objects || [];
const newObjects = Array.isArray(newValue) ? newValue : [];
for (let i = 0; i < Math.max(oldObjects.length, newObjects.length); i++) {
const oldObj = oldObjects[i];
const newObj = newObjects[i] || null;
if (oldObj) {
const simpleFields = ["name", "objectId", "size"];
for (const fieldName of simpleFields) {
const oldField = oldObj[fieldName];
const newField = newObj && typeof newObj === "object" && fieldName in newObj
? newObj[fieldName]
: null;
this.trackFieldHistory(oldField, newField, `objects.${i}.${fieldName}`, updatedBy, historyUpdates);
}
if (oldObj.position) {
const newPosition = newObj && typeof newObj === "object" && "position" in newObj
? newObj.position
: null;
const positionFields = ["x", "y", "z"];
for (const coord of positionFields) {
const oldCoordField = oldObj.position[coord];
const newCoordField = newPosition?.[coord];
this.trackFieldHistory(oldCoordField, newCoordField, `objects.${i}.position.${coord}`, updatedBy, historyUpdates);
}
}
if (oldObj.rotation) {
const newRotation = newObj && typeof newObj === "object" && "rotation" in newObj
? newObj.rotation
: null;
const rotationFields = ["x", "y", "z"];
for (const coord of rotationFields) {
const oldCoordField = oldObj.rotation[coord];
const newCoordField = newRotation?.[coord];
this.trackFieldHistory(oldCoordField, newCoordField, `objects.${i}.rotation.${coord}`, updatedBy, historyUpdates);
}
}
}
}
logger.debug({ investigationId: investigation._id, historyCount: Object.keys(historyUpdates).length }, "Objects array update: tracked history for individual object fields");
continue;
}
if (!fieldPath.includes(".value")) {
continue;
}
const baseFieldPath = fieldPath.replace(".value", "");
let currentValue = null;
let currentField = null;
const pathParts = baseFieldPath.split(".");
let current = investigation;
try {
for (const part of pathParts) {
if (Array.isArray(current) && /^\d+$/.test(part)) {
current = current[parseInt(part, 10)];
}
else if (current &&
typeof current === "object" &&
current !== null &&
part in current) {
current = current[part];
}
else {
current = null;
break;
}
}
currentField = current;
if (currentField &&
typeof currentField === "object" &&
currentField !== null &&
"value" in currentField) {
currentValue = currentField.value;
}
}
catch (error) {
logger.warn({ error, fieldPath }, "Failed to get current value for history");
continue;
}
if (currentValue !== newValue && currentValue !== undefined) {
historyUpdates[baseFieldPath] = {
historyEntry: {
value: currentValue,
updatedBy,
updatedAt: new Date(),
},
fieldPath: baseFieldPath,
};
}
}
return historyUpdates;
}
buildInvestigationSnapshot(investigation) {
// Create a deep copy of the investigation state to ensure it doesn't get mutated
// This captures the OLD state before any updates are applied
const snapshot = JSON.parse(JSON.stringify(investigation.toObject({
depopulate: true,
minimize: false,
getters: false,
virtuals: false,
versionKey: false,
})));
// Remove versioning metadata - these are not part of the investigation state snapshot
delete snapshot._id;
delete snapshot.id;
delete snapshot.__v;
delete snapshot.versions;
delete snapshot.currentVersionIndex;
delete snapshot.lastChangeWithAI;
return snapshot;
}
createVersionEntry(investigation, updatedBy, isAiGenerated = false) {
const snapshot = this.buildInvestigationSnapshot(investigation);
const versions = Array.isArray(investigation.versions) ? investigation.versions : [];
// Get the latest version number (newest version is at the end of the array)
const latestVersionNumber = versions.length > 0 ? (versions[versions.length - 1]?.versionNumber ?? versions.length) : 0;
return {
versionNumber: latestVersionNumber + 1,
snapshot,
updatedBy,
updatedAt: new Date(),
isAiGenerated,
};
}
prepareSnapshotForRestore(snapshot) {
const payload = { ...snapshot };
delete payload._id;
delete payload.id;
delete payload.__v;
delete payload.versions;
return payload;
}
/**
* Updates an investigation using the new API format.
* @category Services
* @param {string} investigationId - The ID of the investigation to update.
* @param {IUpdateInvestigationDto} updateData - The update data in the new format, including string/number fields and step updates.
* @param {object} obj - Optional object used to determine if the update data should be built from scratch.
* @param {boolean} obj.needsBuildUpdate - Whether the update data should be built from scratch.
* @param {Types.ObjectId | null} obj.updatedBy - The user making the change (optional).
* @param {boolean} obj.isAiGenerated - Whether this update is AI-generated (default: false).
* @param {boolean} obj.skipVersioning - Whether to skip versioning (default: false).
* @returns {Promise<IUpdateInvestigationResponse>} - The result of the update operation.
* @throws {Error} If the investigation is not found or is locked.
*/
async updateInvestigation(investigationId, updateData, obj) {
const logger = getLogger();
try {
logger.info({ investigationId, updateData }, "Updating investigation with new format");
// Find the investigation first
const investigation = await Investigation.findById(investigationId);
if (!investigation) {
throw new Error("Investigation not found");
}
const updatedBy = obj?.updatedBy ?? null;
const isAiGenerated = obj?.isAiGenerated ?? false;
// If currentVersionIndex is -1, skip field updates (only send empty fields in response, don't update DB)
const currentVersionIndex = investigation.currentVersionIndex ?? 0;
if (currentVersionIndex === -1) {
logger.info({ investigationId, currentVersionIndex }, "Skipping field updates: currentVersionIndex is -1 (empty state)");
// Return the investigation without updating fields
return {
success: true,
updatedInvestigation: investigation,
};
}
// IMPORTANT: Capture snapshot of OLD state BEFORE any updates are applied
// This ensures we save the state BEFORE the current changes
const oldStateSnapshot = this.buildInvestigationSnapshot(investigation);
// Use builder to create update fields
let updateFields = updateData;
if (obj?.needsBuildUpdate) {
updateFields = InvestigationBuilder.buidlUpdate(updateData, isAiGenerated, investigation);
}
// History saving temporarily disabled
// const historyUpdates = this.saveFieldHistory(investigation, updateFields, updatedBy);
const historyUpdates = {};
const hasUserUpdates = Object.keys(updateFields).length > 0;
// Save snapshot of OLD state in versions - NOT the new/current state
// Store version if:
// 1. lastChangeWithAI !== isAiGenerated (change type switched)
// 2. OR isAiGenerated === true (upcoming change is AI-generated)
let versionEntry = null;
const versions = Array.isArray(investigation.versions) ? investigation.versions : [];
let newVersionIndex = investigation.currentVersionIndex ?? versions.length;
if (hasUserUpdates && !obj?.skipVersioning) {
const lastChangeWithAI = investigation.lastChangeWithAI ?? false;
const upcomingChangeWithAI = isAiGenerated;
// Store version if change type switched OR if upcoming change is AI-generated
const shouldStoreVersion = lastChangeWithAI !== upcomingChangeWithAI || upcomingChangeWithAI;
if (shouldStoreVersion) {
// Use lastChangeWithAI from the investigation (state before this update) to mark the version
// This represents whether the state we're saving was generated by AI
const versionIsAiGenerated = investigation.lastChangeWithAI ?? false;
// Get the latest version number (newest version is at the end of the array)
const latestVersionNumber = versions.length > 0
? (versions[versions.length - 1]?.versionNumber ?? versions.length)
: 0;
// Store the OLD state in versions (snapshot was taken before any updates)
versionEntry = {
versionNumber: latestVersionNumber + 1,
snapshot: oldStateSnapshot, // This is the OLD state, captured before any updates
updatedBy,
updatedAt: new Date(),
isAiGenerated: versionIsAiGenerated, // Use lastChangeWithAI from the old state
};
// After saving old state to versions, currentVersionIndex should be versions.length (current state)
// This will be updated after the version is pushed
newVersionIndex = versions.length + 1; // +1 because we're about to push a new version
}
}
// Separate fields that conflict with $push operations
const setFields = { ...updateFields };
const pushOps = {};
// History saving temporarily disabled - no need to handle object history conflicts
const mongoUpdateOps = {
$set: setFields,
};
if (Object.keys(historyUpdates).length > 0 && !obj?.skipVersioning) {
for (const [baseFieldPath, historyData] of Object.entries(historyUpdates)) {
if (historyData && typeof historyData === "object" && "historyEntry" in historyData) {
const { historyEntry } = historyData;
pushOps[`${baseFieldPath}.history`] = {
$each: [historyEntry],
$position: 0,
};
}
}
}
if (versionEntry) {
// Push the new version to the end of the array (snapshot of state before this update)
pushOps.versions = {
$each: [versionEntry],
$slice: MAX_INVESTIGATION_VERSIONS,
};
// Update currentVersionIndex to point to the new version (current state)
setFields.currentVersionIndex = newVersionIndex;
// Update lastChangeWithAI based on whether this update is AI-generated
setFields.lastChangeWithAI = isAiGenerated;
}
if (Object.keys(pushOps).length > 0) {
mongoUpdateOps.$push = pushOps;
}
let updatedInvestigation = await Investigation.findByIdAndUpdate(investigationId, mongoUpdateOps, { new: true, runValidators: true });
// if (Object.keys(historyUpdates).length > 0 && updatedInvestigation) {
// for (const baseFieldPath of Object.keys(historyUpdates)) {
// const pathParts = baseFieldPath.split(".");
// let field: unknown = updatedInvestigation;
// for (const part of pathParts) {
// if (Array.isArray(field) && /^\d+$/.test(part)) {
// field = field[parseInt(part, 10)];
// } else if (field && typeof field === "object" && field !== null && part in field) {
// field = (field as Record<string, unknown>)[part];
// } else {
// field = null;
// break;
// }
// }
// if (field && typeof field === "object" && field !== null && "history" in field) {
// const fieldWithHistory = field as { history?: unknown[] };
// if (Array.isArray(fieldWithHistory.history) && fieldWithHistory.history.length > 50) {
// fieldWithHistory.history = fieldWithHistory.history.slice(0, 50);
// }
// }
// }
// }
if (updatedInvestigation) {
updatedInvestigation = await updatedInvestigation.save();
}
logger.info({ investigationId }, "Investigation updated successfully with new format");
return {
success: true,
updatedInvestigation: updatedInvestigation?.toObject(),
};
}
catch (error) {
logger.error({ error, investigationId, updateData }, "Failed to update investigation with new format");
throw new Error("Failed to update investigation");
}
}
/**
* Gets the history for a specific field in an investigation.
* @category Services
* @param {string} investigationId - The ID of the investigation.
* @param {string} fieldPath - The path to the field (e.g., "title", "steps.0.title", "objects.0.name").
* @param {number} limit - Maximum number of history entries to return (default: 50).
* @returns {Promise<{fieldPath: string, history: unknown[], totalCount: number}>} The field history.
* @throws {Error} If the investigation is not found.
*/
async getFieldHistory(investigationId, fieldPath, limit = 50) {
const logger = getLogger();
try {
const investigation = await Investigation.findById(investigationId);
if (!investigation) {
throw new Error("Investigation not found");
}
const pathParts = fieldPath.split(".");
let field = investigation;
for (const part of pathParts) {
if (Array.isArray(field) && /^\d+$/.test(part)) {
field = field[parseInt(part, 10)];
}
else if (field && typeof field === "object" && field !== null && part in field) {
field = field[part];
}
else {
field = null;
break;
}
}
if (!field || typeof field !== "object" || field === null || !("history" in field)) {
return {
fieldPath,
history: [],
totalCount: 0,
};
}
const fieldWithHistory = field;
const history = Array.isArray(fieldWithHistory.history) ? fieldWithHistory.history : [];
const limitedHistory = history.slice(0, limit);
return {
fieldPath,
history: limitedHistory,
totalCount: history.length,
};
}
catch (error) {
logger.error({ error, investigationId, fieldPath }, "Failed to get field history");
throw new Error("Failed to get field history");
}
}
/**
* Gets the versions of an investigation.
* @category Services
* @param {string} investigationId - The ID of the investigation.
* @param {number} limit - The number of versions to return (default: 20).
* @returns {Promise<IInvestigationVersionListResponse>} - The investigation versions.
* @throws {Error} If the investigation is not found.
*/
async getInvestigationVersions(investigationId, limit = 20) {
const logger = getLogger();
try {
const investigation = await Investigation.findById(investigationId).select("versions");
if (!investigation) {
throw new Error("Investigation not found");
}
const versions = Array.isArray(investigation.versions) ? investigation.versions : [];
// Get the last N versions (newest are at the end of the array)
const limitedVersions = versions.slice(-limit);
return {
investigationId,
totalCount: versions.length,
versions: limitedVersions.map((entry) => ({
versionNumber: entry.versionNumber,
updatedAt: entry.updatedAt,
updatedBy: entry.updatedBy ? entry.updatedBy.toString() : null,
snapshot: entry.snapshot,
isAiGenerated: entry.isAiGenerated ?? false,
})),
};
}
catch (error) {
logger.error({ error, investigationId }, "Failed to get investigation versions");
throw new Error("Failed to get investigation versions");
}
}
/**
* Undo investigation - decrement currentVersionIndex and return investigation at that version
* @category Services
* @param {string} investigationId - The ID of the investigation.
* @returns {Promise<IInvestigation>} - The investigation at the new index.
* @throws {Error} If the investigation is not found or if the index is out of bounds.
*/
async undoInvestigation(investigationId) {
const logger = getLogger();
try {
const investigation = await Investigation.findById(investigationId);
if (!investigation) {
throw new Error("Investigation not found");
}
const currentIndex = investigation.currentVersionIndex ?? 0;
const versions = Array.isArray(investigation.versions) ? investigation.versions : [];
// Versions are stored with oldest first, newest last
// Index -1 = special empty state (cannot undo from here)
// Index 0 = can undo to go to -1 (even if no versions exist)
// Index versions.length = current state (not in versions array)
// Can undo from current state (index = versions.length) to go to the previous saved version
// Can undo from index 0 to go to index -1 (empty state) - even if no versions exist
// Can't undo if already at index -1 (empty state)
// Undo decrements the index (goes to a lower index number)
if (currentIndex === -1) {
throw new Error("Cannot undo: already at the oldest saved version");
}
// Decrement index (go to an older saved version or empty state)
// If at index 2 (current state, versions.length=2), go to index 1 (versions[1] = newest saved version)
// If at index 1, go to index 0 (versions[0] = oldest saved version)
// If at index 0, go to index -1 (empty state)
const newIndex = currentIndex - 1;
logger.info({ investigationId, currentIndex, newIndex, versionsLength: versions.length }, "Undoing investigation: decrementing currentVersionIndex");
// Update currentVersionIndex
await Investigation.findByIdAndUpdate(investigationId, { $set: { currentVersionIndex: newIndex } }, { new: true });
// Get and return the investigation at the new index
const restoredInvestigation = await this.getInvestigationById(investigationId);
logger.info({
investigationId,
newIndex,
restoredVersionIndex: restoredInvestigation.currentVersionIndex,
}, "Undo complete: investigation retrieved");
return restoredInvestigation;
}
catch (error) {
logger.error({ error, investigationId }, "Failed to undo investigation");
throw error;
}
}
/**
* Redo investigation - increment currentVersionIndex and return investigation at that version
* @category Services
* @param {string} investigationId - The ID of the investigation.
* @returns {Promise<IInvestigation>} - The investigation at the new index.
* @throws {Error} If the investigation is not found or if the index is out of bounds.
*/
async redoInvestigation(investigationId) {
const logger = getLogger();
try {
const investigation = await Investigation.findById(investigationId);
if (!investigation) {
throw new Error("Investigation not found");
}
const currentIndex = investigation.currentVersionIndex ?? 0;
const versions = Array.isArray(investigation.versions) ? investigation.versions : [];
// Versions are stored with oldest first, newest last
// Index -1 = special empty state - can redo to go to index 0 (even if no versions exist)
// Index 0 = versions[0] (oldest saved version)
// Index versions.length = current state (not in versions array)
// Can't redo if already at index versions.length (current state, newest version)
// Redo increments the index (goes to a higher index number, towards current state)
const maxIndex = versions.length;
// Allow redo from -1 to 0 even if no versions exist
// Otherwise, require versions to exist and not be at maxIndex
if (currentIndex === -1) {
// Can redo from -1 to 0, proceed
}
else if (currentIndex >= maxIndex || versions.length === 0) {
throw new Error("Cannot redo: already at the current state (newest version)");
}
// Increment index (go to a newer version, towards current state)
// If at index -1, go to index 0 (oldest saved version)
// If at index 0, go to index 1 (newer saved version)
// If at index 1, go to index 2 (newer saved version)
// If at index versions.length - 1, go to index versions.length (current state, newest)
const newIndex = currentIndex + 1;
// Update currentVersionIndex
await Investigation.findByIdAndUpdate(investigationId, { $set: { currentVersionIndex: newIndex } }, { new: true });
// Get and return the investigation at the new index
return await this.getInvestigationById(investigationId);
}
catch (error) {
logger.error({ error, investigationId }, "Failed to redo investigation");
throw error;
}
}
/**
* Ensures that the author of a given investigation is set.
*
* If the investigation exists and does not already have a valid author, this method sets the author
* to the provided `authorId` and updates the `dateModified` field.
* @category Services
* @param {string} investigationId - The ID of the investigation to check.
* @param {Types.ObjectId} authorId - The ObjectId of the author to set.
* @returns {Promise<IInvestigation | null>} - The updated investigation object, or `null` if the investigation does not exist or already has a valid author.
* @throws {Error} If there is an error during the process of fetching or saving the investigation.
*/
async ensureInvestigationAuthor(investigationId, authorId) {
const logger = getLogger();
try {
const investigation = await Investigation.findById(investigationId);
if (!investigation) {
return null;
}
const currentAuthor = investigation.metadata?.author;
if (currentAuthor && Types.ObjectId.isValid(currentAuthor.toString())) {
return null;
}
if (!investigation.metadata) {
investigation.metadata = {};
}
investigation.metadata.author = authorId;
investigation.metadata.dateModified = new Date();
const saved = await investigation.save();
return saved.toObject();
}
catch (error) {
logger.error({ error, investigationId, authorId }, "Failed to ensure investigation author");
throw new Error("Failed to ensure investigation author");
}
}
/**
* Adds a contradiction to a specific field in an investigation.
* @category Services
* @param {Partial<IInvestigation>} investigation - The investigation object to update.
* @param {IContradictionInput} contradiction - The details of the contradiction to add.
* @returns {Partial<IInvestigation>} - The investigation object with the updated contradiction.
* @throws {Error} - If the field is not found in the investigation.
*/
addContradictionInPlace(investigation, contradiction) {
const logger = getLogger();
try {
logger.info({ contradiction }, "Adding contradiction to investigation field");
// Get the field to update - properly typed as IEditableField
const fieldToUpdate = investigation[contradiction.fieldName];
if (!fieldToUpdate) {
logger.warn({ fieldName: contradiction.fieldName }, "Field not found in investigation");
throw new Error(`Field '${contradiction.fieldName}' not found in investigation`);
}
// Update the field in place with contradiction information
fieldToUpdate.isContradicting = true;
fieldToUpdate.targetFieldName = contradiction.targetFieldName;
fieldToUpdate.contradictionReason = contradiction.contradictionReasoning;
logger.info({
fieldName: contradiction.fieldName,
targetFieldName: contradiction.targetFieldName,
}, "Contradiction added successfully");
// Return the updated investigation (mutated in place)
return investigation;
}
catch (error) {
logger.error({ error, contradiction }, "Failed to add contradiction to investigation");
throw new Error("Failed to add contradiction to investigation");
}
}
/**
* Retrieves investigations with pagination, grouped by curriculum.
* @category Services
* @param {IInvestigationSearchDto} searchDto - The search and pagination parameters.
* @returns {Promise<IInvestigationListResponseDto>}- The paginated investigations grouped by curriculum.
*/
async getInvestigations(searchDto) {
const logger = getLogger();
try {
logger.info({ searchDto }, "Getting investigations with pagination and grouping by curriculum");
// Set default values
const limit = searchDto.limit || 2;
const offset = searchDto.offset || 0;
const sortBy = searchDto.sortBy || "createdAt";
const sortOrder = searchDto.sortOrder || "desc";
// Build sort object
const sort = {};
sort[sortBy] = sortOrder === "asc" ? 1 : -1;
// Build query with search functionality
const query = {};
let hasSearch = false;
let searchTerm = "";
let escapedSearchTerm = "";
let searchRegex = null;
// If search parameter is provided, search across multiple fields
if (searchDto.search && searchDto.search.trim()) {
hasSearch = true;
searchTerm = searchDto.search.trim();
// Escape special regex characters and make case-insensitive
escapedSearchTerm = searchTerm.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
searchRegex = new RegExp(escapedSearchTerm, "i");
// Define all fields to search across
const searchFields = [
// Basic fields
"title.value",
"curriculum.value",
"unitNumberAndTitle.value",
"lessonNumberAndTitle.value",
"grade.value",
// Content fields
"objectives.value",
"goals.value",
"analyticalFacts.value",
"ngss.value",
"day.value",
// Steps fields
"steps.title.value",
"steps.descriptionEn.value",
"steps.desiredOutcome.value",
"steps.alternativeOutcome.value",
"steps.name",
// Objects fields
"objects.name.value",
"objects.objectId.value",
];
// Build $or query from field array
query.$or = searchFields.map((field) => ({
[field]: { $regex: searchRegex },
}));
}
// Apply additional filters if provided
if (searchDto.status) {
query.status = searchDto.status;
}
if (searchDto.grade) {
query["grade.value"] = searchDto.grade;
}
if (searchDto.curriculum) {
query["curriculum.value"] = searchDto.curriculum;
}
if (searchDto.author) {
query["metadata.author"] = new Types.ObjectId(searchDto.author);
}
if (searchDto.isLocked !== undefined) {
query.isLocked = searchDto.isLocked;
}
// Single optimized aggregation pipeline that:
// 1. Gets total count for pagination
// 2. Fetches investigations with pagination
// 3. Joins with URL data in one query
const aggregationResult = await Investigation.aggregate([
// Stage 1: Match investigations (apply any filters)
{ $match: query },
// Stage 2: Add relevance score if searching
...(hasSearch ? [buildRelevanceScoreStage(escapedSearchTerm)] : []),
// Stage 3: Use $facet to get both total count and paginated results
{
$facet: {
// Get total count for pagination
totalCount: [{ $count: "count" }],
// Get paginated results with URL data
paginatedResults: [
// Sort by relevance score first (if searching), then by original sort criteria
{
$sort: hasSearch ? { relevanceScore: -1, ...sort } : sort,
},
{ $skip: offset },
{ $limit: limit },
// Lookup URL data and aggregate in one step
{
$lookup: {
from: "urls",
localField: "_id",
foreignField: "investigationId",
as: "urls",
},
},
// Add computed URL fields directly in the pipeline
{
$addFields: {
views: { $sum: "$urls.counters.visitsTotal" },
completions: { $sum: "$urls.counters.completionsTotal" },
urls: { $size: "$urls" },
},
},
// Project only needed fields
{
$project: {
_id: 1,
status: 1,
titleValue: "$title.value",
gradeValue: "$grade.value",
curriculumValue: "$curriculum.value",
views: 1,
completions: 1,
urls: 1,
author: "$metadata.author",
editors: "$metadata.editors",
unit: "$unitNumberAndTitle.value",
lesson: "$lessonNumberAndTitle.value",
day: "$day.value",
createdAt: "$createdAt",
updatedAt: "$updatedAt",
publishedAt: "$metadata.dateOfPublishing",
sentToDevelopmentAt: "$metadata.dateOfDevelopmentDelivery",
activity: null,
timeInDevelopment: "$metadata.dateOfDevelopment",
},
},
],
},
},
]);
// Extract results from aggregation with proper null checking
const total = aggregationResult[0]?.totalCount?.[0]?.count || 0;
const investigations = aggregationResult[0]?.paginatedResults || [];
// Debug logging to verify aggregation results
logger.info({
total,
investigationsCount: investigations.length,
sampleInvestigation: investigations[0] || null,
}, "Aggregation results debug info");
// Convert to response DTOs
const investigationListItems = investigations.map((investigation) => ({
id: investigation._id.toString(),
status: investigation.status,
title: investigation.titleValue || "",
grade: investigation.gradeValue || "",
curriculum: investigation.curriculumValue || "",
views: investigation.views || 0,
completions: investigation.completions || 0,
urls: investigation.urls || 0,
author: {
id: investigation.author ? investigation.author.toString() : null,
name: null,
email: null,
avatar: null,
},
editors: investigation.editors || [],
subject: subjectsFactory[investigation.unit?.split(" ")[1]?.split(".")[0] ?? ""] || null,
unit: investigation.unit || null,
lesson: investigation.lesson || null,
day: investigation.day ?? null,
createdAt: investigation.createdAt || null,
updatedAt: investigation.updatedAt || null,
publishedAt: investigation.publishedAt || null,
sentToDevelopmentAt: investigation.sentToDevelopmentAt || null,
activity: investigation.activity || null,
timeInDevelopment: investigation.timeInDevelopment || null,
}));
// Group investigations by curriculum
// const sectionsMap = new Map<string, IInvestigationListItemDto[]>();
// investigationListItems.forEach((item) => {
// const curriculum = item.curriculum || "Unknown";
// if (!sectionsMap.has(curriculum)) {
// sectionsMap.set(curriculum, []);
// }
// sectionsMap.get(curriculum)!.push(item);
// });
//
// // Convert map to sections array
// const sections = Array.from(sectionsMap.entries()).map(([title, items]) => ({
// title,
// items,
// }));
// Calculate pagination metadata
const currentPage = Math.floor(offset / limit) + 1;
const totalPages = Math.ceil(total / limit);
const hasNextPage = currentPage < totalPages;
const hasPrevPage = currentPage > 1;
logger.info({
total,
limit,
offset,
currentPage,
totalPages,
itemsCount: investigationListItems.length,
totalItems: investigationListItems.length,
}, "Investigations retrieved and grouped successfully");
return {
items: investigationListItems,
pagination: {
total,
limit,
offset,
currentPage,
totalPages,
hasNextPage,
hasPrevPage,
},
};
}
catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error);
logger.error({ error, errorMessage, searchDto }, "Failed to get investigations");
throw new Error(`Failed to get investigations: ${errorMessage}`);
}
}
/**
* Generates an investigation without steps and objects based on the provided metadata.
* If the investigation is not found, it throws an error.
* @category Services
* @param {string} message - The user's message from which the lesson must be detected.
* @param {string} history - The chat history between the user and the assistant.
* @param {string} investigationMetadata - The metadata of the investigation.
* @returns {Promise<IAssistantInvestigationFormat | null>} - The generated investigation without steps and objects.
* @throws {Error} - If the investigation is not found.
*/
async generateInvestigationWithoutStepsAndObjects(message, history, investigationMetadata) {
const logger = getLogger();
try {
logger.info("Generating investigation without steps and objects.");
const prompt = await Prompt.findOne({
name: "generate_investigation_without_steps_and_objects",
});
if (!prompt) {
throw new Error("generate_investigation_without_steps_and_objects prompt not found.");
}
const promptTemplate = prompt.template
.replace("{guide}", investigationMetadata)
.replace("{history}", history || "-");
let retry = 0;
let response = null;
while (retry < 3) {
response = (await this.aiProcessor.fetchLLMResponse(promptTemplate, AssistantInvestigationFormat));
if (response.title === undefined ||
response.title === null ||
response.curriculum === undefined ||
response.curriculum === null ||
response.unitNumberAndTitle === undefined ||
response.unitNumberAndTitle === null ||
response.grade === undefined ||
response.grade === null ||
response.lessonNumberAndTitle === undefined ||
response.lessonNumberAndTitle === null ||
response.objectives === undefined ||
response.objectives === null ||
response.ngss === undefined ||
response.ngss === null ||
response.analyticalFacts === undefined ||
response.analyticalFacts === null ||
response.goals === undefined ||
response.goals === null ||
response.day === undefined ||
response.day === null) {
retry += 1;
}
else {
break;
}
}
if (retry === 3 || !response) {
throw new Error(`Failed to generate investigation without steps and objects from message: ${message}. Response is: ${JSON.stringify(response)}.`);
}
logger.info(`Response for generating investigation without steps and objects is: ${JSON.stringify(response)}`);
return response;
}
catch (error) {
logger.error({
message: `Failed to generate investigation without steps and objects for message: ${message}.`,
error: error instanceof Error ? error.message : "Unknown error",
});
throw error;
}
}
/**
* Generates objects of investigation based on the provided metadata and plain investigation.
* If the objects are not found, it throws an error.
* @category Services
* @param {string} message - The user's message from which the lesson must be detected.
* @param {string} history - The chat history between the user and the assistant.
* @param {string} investigationMetadata - The metadata of the investigation.
* @param {string} investigation - The plain investigation without steps and objects with yaml format.
* @returns {Promise<IAssistantInvestigationFormat | null>} - The generated objects of the provided investigation.
* @throws {Error} - If the objects are not found.
*/
async generateInvestigationObjects(message, history, investigationMetadata, investigation) {
const logger = getLogger();
try {
logger.info("Generating objects of the provided investigation.");
const prompt = await Prompt.findOne({
name: "generate_investigation_objects",
});
if (!prompt) {
throw new Error("generate_investigation_objects prompt not found.");
}
const promptTemplate = prompt.template
.replace("{guide}", investigationMetadata)
.replace("{history}", history || "-")
.replace("{investigation}", investigation || "-");
let retry = 0;
let response = null;
while (retry < 3) {
response = (await this.aiProcessor.fetchLLMResponse(promptTemplate, AssistantInvestigationFormat));
if (response.objects === undefined || response.objects === null) {
retry += 1;
}
else {
break;
}
}
if (retry === 3 || !response) {
throw new Error(`Failed to generate objects of the provided investigation from message: ${message}. Response is: ${JSON.stringify(response)}.`);
}
logger.info(`Response for generating objects of the provided investigation is: ${JSON.stringify(response)}`);
return response;
}
catch (error) {
logger.error({
message: `Failed to generate objects of the provided investigation for message: ${message}.`,
error: error instanceof Error ? error.message : "Unknown error",
});
throw error;
}
}
/**
* Generates steps of investigation based on the provided metadata and the provided investigation.
* If the steps are not found, it throws an error.
* @category Services
* @param {string} message - The user's message from which the lesson must be detected.
* @param {string} history - The chat history between the user and the assistant.
* @param {string} investigationMetadata - The metadata of the investigation.
* @param {string} investigation - The provided investigation without steps with yaml format.
* @returns {Promise<IAssistantInvestigationFormat | null>} - The generated steps of the provided investigation.
* @throws {Error} - If the steps are not found.
*/
async generateInvestigationSteps(message, history, investigationMetadata, investigation) {
const logger = getLogger();
try {
logger.info("Generating steps of the provided investigation.");
const prompt = await Prompt.findOne({
name: "generate_investigation_steps",
});
if (!prompt) {
throw new Error("generate_investigation_steps prompt not found.");
}
const promptTemplate = prompt.template
.replace("{guide}", investigationMetadata)
.replace("{history}", history || "-")
.replace("{investigation}", investigation || "-");
let retry = 0;
let response = null;
while (retry < 3) {
response = (await this.aiProcessor.fetchLLMResponse(promptTemplate, AssistantInvestigationFormat));
if (response.steps === undefined || response.steps === null) {
retry += 1;
}
else {
break;
}
}
if (retry === 3 || !response) {
throw new Error(`Failed to generate steps of the provided investigation from message: ${message}. Response is: ${JSON.stringify(response)}.`);
}
logger.info(`Response for generating steps of the provided investigation is: ${JSON.stringify(response)}`);
return response;
}
catch (error) {
logger.error({
message: `Failed to generate steps of the provided investigation for message: ${message}.`,
error: error instanceof Error ? error.message : "Unknown error",
});
throw error;
}
}
/**
* Generates an entire investigation based on the provided metadata.
* If the investigation is not found, it throws an error.
* @category Services
* @param {string} message - The user's message from which the lesson must be detected.
* @param {FormattedHistory} history - The chat history between the user and the assistant.
* @param {string} investigationMetadata - The metadata of the investigation.
* @returns {Promise<IAssistantInvestigationFormat | null>} - The generated investigation.
* @throws {Error} - If the investigation is not found.
*/
async generateInvestigation(message, history, investigationMetadata) {
const logger = getLogger();
try {
logger.info("Generating investigation.");
let chatHistory = [];
if (history) {
chatHistory.push(...history);
}
chatHistory.push({ role: "user", content: message });
const yaml = yamlModule;
const investigationMetadataObject = investigationMetadata.toObject({
versionKey: false,
transform: (doc, ret) => {
delete ret._id;
return ret;
},
});
const investigationMetadataYaml = yaml.dump(investigationMetadataObject);
let investigation = (await this.generateInvestigationWithoutStepsAndObjects(message, yaml.dump(chatHistory), investigationMetadataYaml));
const objects = await this.generateInvestigationObjects(message, yaml.dump(chatHistory), investigationMetadataYaml, yaml.dump(investigation));
investigation.objects = objects?.objects;
const steps = await this.generateInvestigationSteps(message, yaml.dump(chatHistory), investigationMetadataYaml, yaml.dump(investigation));
investigation.steps = steps?.steps;
logger.info(`Response for generating investigation is: ${JSON.stringify(investigation)}`);
return investigation;
}
catch (error) {
logger.error({
message: `Failed to generate investigation for message: ${message}.`,
error: error instanceof Error ? error.message : "Unknown error",
});
throw error;
}
}
// src/services/investigation.service.ts (lines 1106-1237)
/**
* Clones the current investigation by adding a "copy_N" postfix to its identifier.
* @category Services
* @param {string} investigationId - The ID of the current investigation to clone.
* @returns {Promise<IInvestigation>} - The cloned investigation object.
* @throws {Error} - If an error occurs during the cloning process.
*/
async cloneInvestigation(investigationId) {
const logger = getLogger();
try {
logger.info({ investigationId }, "Cloning investigation");
// Get the original investigation first
const originalInvestigation = await this.getInvestigationById(investigationId);
logger.debug({ originalInvestigationId: originalInvestigation._id }, "Original investigation found");
// Create a deep copy of the investigation data with proper typing
const investigationData = originalInvestigation.toObject();
// Remove the _id and __v fields to create a new document
delete investigationData._id;
delete investigationData.__v;
delete investigationData.createdAt;
delete investigationData.updatedAt;
// Remove all aiGeneratedValue fields recursively
this.removeAiGeneratedValues(investigationData);
// Handle title postfix logic
const originalTitle = investigationData.title?.value || "";
const copyRegex = / copy_(\d+)$/;
const copyMatch = originalTitle.match(copyRegex);
let newTitle;
if (copyMatch && copyMatch[1]) {
// If title already has copy postfix, increment the number
const currentCopyNumber = parseInt(copyMatch[1], 10);
const baseTitle = originalTitle.replace(copyRegex, "");
newTitle = `${baseTitle} copy_${currentCopyNumber + 1}`;
}
else {
// If no copy postfix, count existing clones and add copy_1 (or next number)
const existingClonesCount = await Investigation.countDocuments({
"metadata.sourceInvestigation": originalInvestigation._id,
});
newTitle = `${originalTitle} copy_${existingClonesCount + 1}`;
}
// Update the title
investigationData.title.value = newTitle;
// Update metadata for the clone - merge original metadata with specific updates
investigationData.metadata.sourceInvestigation =
originalInvestigation._id;
investigationData.metadata.dateOfCreation = new Date();
// TODO: Later we will also update the author field to the user who cloned it
// Reset status to draft
investigationData.status = "draft_incomplete";
investigationData.isLocked = false;
investigationData.lockedBy = null;
investigationData.lockedAt = null;
// Create new investigation document
const clonedInvestigation = new Investigation(investigationData);
const savedInvestigation = await clonedInvestigation.save();
logger.info({
originalInvestigationId: originalInvestigation._id,
clonedInvestigationId: savedInvestigation._id,
}, "Investigation cloned successfully");
return savedInvestigation;
}
catch (error) {
// Enhanced error logging for better debugging
const errorMessage = error instanceof Error ? error.message : String(error);
const errorStack = error instanceof Error ? error.stack : undefined;
const errorName = error instanceof Error ? error.name : "UnknownError";
logger.error({
error: {
message: errorMessage,
name: errorName,
stack: errorStack,
originalError: error,
},
investigationId,
}, "Failed to clone investigation");
throw new Error(`Failed to clone investigation: ${errorMessage}`);
}
}
/**
* Deletes the data of a given investigation.
* @category Services
* @param {IInvestigation} investigation - The investigation to clear.
* @returns {IInvestigation} - The investigation object with its values emptied.
* @throws {Error} - If an error occurs during the deletion process.
*/
deleteInvestigationData(investigation) {
const logger = getLogger();
try {
logger.info("Deleting investigation Data.");
investigation.status = InvestigationStatus.DRAFT_INCOMPLETE;
investigation.curriculum.value = null;
investigation.curriculum.aiGeneratedValue = null;
investigation.unitNumberAndTitle.value = null;
investigation.unitNumberAndTitle.aiGeneratedValue = null;
investigation.grade.value = null;
investigation.grade.aiGeneratedValue = null;
investigation.lessonNumberAndTitle.value = null;
investigation.lessonNumberAndTitle.aiGeneratedValue = null;
investigation.day.value = null;
investigation.day.aiGeneratedValue = null;
investigation.title.value = investigation.title.value + " (empty)";
investigation.title.aiGeneratedValue = investigation.title.aiGeneratedValue + " (empty)";
investigation.ngss.value = null;
investigation.ngss.aiGeneratedValue = null;
investigation.objectives.value = null;
investigation.objectives.aiGeneratedValue = null;
investigation.analyticalFacts.value = null;
investigation.analyticalFacts.aiGeneratedValue = null;
investigation.goals.value = null;
investigation.goals.aiGeneratedValue = null;
investigation.steps = [];
investigation.objects = [];
return investigation;
}
catch (error) {
throw error;
}
}
/**
* Permanently deletes an investigation and its related chat history.
* @category Services
* @param {string} investigationId - The ID of the investigation to delete.
* @returns {Promise<IDeleteInvestigationByIdResponse>} - An object summarizing the deletion status of the investigation and related chats.
* @throws {Error} - If an error occurs during the deletion process.
*/
async deleteInvestigationById(investigationId) {
const logger = getLogger();
try {
logger.info({ investigationId }, "Deleting investigation and related chat history");
const objectId = new Types.ObjectId(investigationId);
const [deletedInvestigation, chatDeleteResult] = await Promise.all([
Investigation.findByIdAndDelete(objectId),
Chat.deleteMany({ investigationId: objectId }),
]);
const deletedChats = chatDeleteResult.deletedCount ?? 0;
const investigationDeleted = Boolean(deletedInvestigation);
logger.info({ investigationId, investigationDeleted, deletedChats }, "Deletion completed");
return {
investigationDeleted,
deletedChats,
};
}
catch (error) {
logger.error({ error, investigationId }, "Failed to delete investigation and chat");
throw error;
}
}
/**
* Updates investigation by adding new version.
* @category Services
* @param {IInvestigation} oldInvestigation - The latest investigation version.
* @param {IInvestigation} investigation - The entire investigation model.
* @param {ObjectId | null} userId - The ID of the user.
* @param {boolean} isUserChangeExist - Whether the change was made by the user (true) or AI (false). Defaults to false.
* @returns {Promise<IInvestigation>} - An investigation updated with new version.
* @throws {Error} - If an error occurs during the update process.
*/
async updateInvestigationByVersion(oldInvestigation, investigation, userId, isUserChangeExist = false) {
const logger = getLogger();
try {
// IMPORTANT: Capture all versioning data from OLD state BEFORE saving
// This ensures we save the snapshot of the state BEFORE the AI-generated changes
// Create a deep copy of the old state to ensure it doesn't get mutated
const oldStateSnapshot = JSON.parse(JSON.stringify(oldInvestigation));
const currentVersionIndex = oldInvestigation.currentVersionIndex ?? 0;
// Remove versioning metadata - these are not part of the investigation state snapshot
delete oldStateSnapshot._id;
delete oldStateSnapshot.id;
delete oldStateSnapshot.versions;
delete oldStateSnapshot.currentVersionIndex;
delete oldStateSnapshot.lastChangeWithAI;
// Capture versioning metadata from OLD investigation state (before save)
const versions = Array.isArray(oldInvestigation.versions) ? oldInvestigation.versions : [];
const lastChangeWithAI = oldInvestigation.lastChangeWithAI ?? false;
const versionIsAiGenerated = oldInvestigation.lastChangeWithAI ?? false;
const updatedBy = userId ? new Types.ObjectId(userId.toString()) : null;
// Use isUserChangeExist to determine if change is AI-generated
// If user change exists, it's NOT AI-generated, so upcomingChangeWithAI = false
// If no user change exists, it IS AI-generated, so upcomingChangeWithAI = true
const upcomingChangeWithAI = !isUserChangeExist;
logger.info({
upcomingChangeWithAI,
lastChangeWithAI,
isUserChangeExist,
});
// Store version if:
// 1. lastChangeWithAI !== isAiGenerated (change type switched)
// 2. OR isAiGenerated === true (upcoming change is AI-generated)
// Store version if change type switched OR if upcoming change is AI-generated
const shouldStoreVersion = lastChangeWithAI !== upcomingChangeWithAI || upcomingChangeWithAI;
// STEP 3: Store the old state snapshot in versions BEFORE saving the new investigation
if (shouldStoreVersion) {
// Get the latest version number (newest version is at the end of the array)
const latestVersionNumber = versions.length > 0
? (versions[versions.length - 1]?.versionNumber ?? versions.length)
: 0;
// Store the OLD state in versions (snapshot was taken before any updates)
const versionEntry = {
versionNumber: latestVersionNumber + 1,
snapshot: oldStateSnapshot, // This is the OLD state, captured before any updates
updatedBy,
updatedAt: new Date(),
isAiGenerated: versionIsAiGenerated, // Use lastChangeWithAI from the old state
};
const newVersionIndex = (currentVersionIndex ?? 0) + 1;
investigation.versions.push(versionEntry);
investigation.currentVersionIndex = newVersionIndex;
}
investigation.lastChangeWithAI = upcomingChangeWithAI;
await investigation.save();
return investigation;
}
catch (error) {
logger.error("Failed to update investigation");
throw error;
}
}
/**
* Regenerates text fields (without steps and objects) based on a specific field value.
* This is Step 1 of the 3-step regeneration process.
* @category Services
* @param {string} fieldName - The name of the field to base regeneration on.
* @param {string} fieldValue - The value of the field to base regeneration on.
* @param {string} investigation - The current investigation in YAML format.
* @param {string} investigationMetadata - The metadata/guide in YAML format.
* @param {string} [history] - Chat history in YAML format (optional, defaults to "-").
* @param {object} [contradictionInfo] - Optional contradiction information.
* @param {string | null} [contradictionInfo.contradictionReason] - Reason for contradiction.
* @param {string | null} [contradictionInfo.targetFieldName] - Target field name for contradiction.
* @param {boolean} [contradictionInfo.isContradicting] - Whether the field is contradicting.
* @returns {Promise<IAssistantInvestigationFormat | null>} - The regenerated text fields only.
* @throws {Error} If the prompt is not found or regeneration fails.
*/
async regenerateOtherFieldsWithoutStepsAndObjects(fieldName, fieldValue, investigation, investigationMetadata, history = "-", contradictionInfo) {
const logger = getLogger();
try {
logger.info(`Regenerating text fields (without steps and objects) based on ${fieldName}`);
// Try to use dedicated prompt, fallback to existing prompt
let prompt = await Prompt.findOne({
name: "regenerate_other_fields_text_only",
});
if (!prompt) {
prompt = await Prompt.findOne({
name: "generate_investigation_without_steps_and_objects",
});
if (!prompt) {
throw new Error("generate_investigation_without_steps_and_objects prompt not found.");
}
}
// Build prompt template - replace ALL occurrences of placeholders
let promptTemplate = prompt.template
.replaceAll("{guide}", investigationMetadata)
.replaceAll("{history}", history || "-")
.replaceAll("{fieldName}", fieldName)
.replaceAll("{fieldValue}", fieldValue);
// Only replace contradiction placeholders if there's actually a contradiction
if (contradictionInfo && contradictionInfo.isContradicting) {
promptTemplate = promptTemplate
.replaceAll("{contradictionReason}", contradictionInfo.contradictionReason || "-")
.replaceAll("{targetFieldName}", contradictionInfo.targetFieldName || "-")
.replaceAll("{isContradicting}", "true");
}
else {
// No contradiction - replace with empty/neutral values
promptTemplate = promptTemplate
.replaceAll("{contradictionReason}", "")
.replaceAll("{targetFieldName}", "")
.replaceAll("{isContradicting}", "false");
}
// If the prompt has {investigation} placeholder, replace ALL occurrences
if (promptTemplate.includes("{investigation}")) {
promptTemplate = promptTemplate.replaceAll("{investigation}", investigation);
}
let retry = 0;
let response = null;
while (retry < 3) {
response = (await this.aiProcessor.fetchLLMResponse(promptTemplate, AssistantInvestigationFormat));
if (!response) {
retry += 1;
continue;
}
const requiredTextFields = [
"title",
"curriculum",
"unitNumberAndTitle",
"grade",
"lessonNumberAndTitle",
"objectives",
"ngss",
"analyticalFacts",
"goals",
"day",
];
const hasMissingFields = requiredTextFields.some((field) => {
const value = response[field];
return value === undefined || value === null;
});
if (hasMissingFields) {
retry += 1;
}
else {
break;
}
}
if (retry === 3 || !response) {
throw new Error(`Failed to regenerate text fields based on ${fieldName}. Response is: ${JSON.stringify(response)}.`);
}
logger.info(`Response for regenerating text fields is: ${JSON.stringify(response)}`);
return response;
}
catch (error) {
logger.error({
message: `Failed to regenerate text fields based on ${fieldName}.`,
error: error instanceof Error ? error.message : "Unknown error",
});
throw error;
}
}
/**
* Regenerates objects based on regenerated text fields.
* This is Step 2 of the 3-step regeneration process.
* @category Services
* @param {string} fieldName - The name of the field to base regeneration on.
* @param {string} fieldValue - The value of the field to base regeneration on.
* @param {string} investigation - The regenerated investigation from Step 1 (YAML format).
* @param {string} investigationMetadata - The metadata/guide in YAML format.
* @param {string} [history] - Chat history in YAML format (optional, defaults to "-").
* @param {object} [contradictionInfo] - Optional contradiction information.
* @param {string | null} [contradictionInfo.contradictionReason] - Reason for contradiction.
* @param {string | null} [contradictionInfo.targetFieldName] - Target field name for contradiction.
* @param {boolean} [contradictionInfo.isContradicting] - Whether the field is contradicting.
* @returns {Promise<IAssistantInvestigationFormat | null>} - The regenerated objects only.
* @throws {Error} If the prompt is not found or regeneration fails.
*/
async regenerateOtherFieldsObjects(fieldName, fieldValue, investigation, investigationMetadata, history = "-", contradictionInfo) {
const logger = getLogger();
try {
logger.info(`Regenerating objects based on ${fieldName}`);
// Try to use dedicated prompt, fallback to existing prompt
let prompt = await Prompt.findOne({
name: "regenerate_other_fields_objects",
});
if (!prompt) {
prompt = await Prompt.findOne({
name: "generate_investigation_objects",
});
if (!prompt) {
throw new Error("generate_investigation_objects prompt not found.");
}
}
// Build prompt template - replace ALL occurrences of placeholders
let promptTemplate = prompt.template
.replaceAll("{guide}", investigationMetadata)
.replaceAll("{history}", history || "-")
.replaceAll("{fieldName}", fieldName)
.replaceAll("{fieldValue}", fieldValue);
// Only replace contradiction placeholders if there's actually a contradiction
if (contradictionInfo && contradictionInfo.isContradicting) {
promptTemplate = promptTemplate
.replaceAll("{contradictionReason}", contradictionInfo.contradictionReason || "-")
.replaceAll("{targetFieldName}", contradictionInfo.targetFieldName || "-")
.replaceAll("{isContradicting}", "true");
}
else {
// No contradiction - replace with empty/neutral values
promptTemplate = promptTemplate
.replaceAll("{contradictionReason}", "")
.replaceAll("{targetFieldName}", "")
.replaceAll("{isContradicting}", "false");
}
// If the prompt has {investigation} placeholder, replace ALL occurrences
if (promptTemplate.includes("{investigation}")) {
promptTemplate = promptTemplate.replaceAll("{investigation}", investigation);
}
try {
const parsedInvestigation = yamlModule.load(investigation);
const existingObjectsCount = parsedInvestigation?.objects?.length || 0;
if (existingObjectsCount > 0) {
const objectCountInstruction = `\n\n### CRITICAL REQUIREMENT:\nYou MUST return EXACTLY ${existingObjectsCount} objects in your response. The investigation context shows ${existingObjectsCount} existing objects, and you MUST regenerate and return ALL ${existingObjectsCount} objects. Do NOT return fewer objects.\n`;
promptTemplate = promptTemplate + objectCountInstruction;
logger.info({
fieldName,
existingObjectsCount,
}, "Added explicit instruction to return all objects");
}
}
catch (parseError) {
logger.warn({ fieldName, error: parseError }, "Failed to parse investigation YAML to count objects, continuing without object count instruction");
}
let retry = 0;
let response = null;
while (retry < 3) {
response = (await this.aiProcessor.fetchLLMResponse(promptTemplate, AssistantInvestigationFormat));
// Validate response has objects
if (response.objects === undefined || response.objects === null) {
retry += 1;
}
else {
break;
}
}
if (retry === 3 || !response) {
throw new Error(`Failed to regenerate objects based on ${fieldName}. Response is: ${JSON.stringify(response)}.`);
}
logger.info(`Response for regenerating objects is: ${JSON.stringify(response)}`);
return response;
}
catch (error) {
logger.error({
message: `Failed to regenerate objects based on ${fieldName}.`,
error: error instanceof Error ? error.message : "Unknown error",
});
throw error;
}
}
/**
* Regenerates steps based on regenerated text fields and objects.
* This is Step 3 of the 3-step regeneration process.
* @category Services
* @param {string} fieldName - The name of the field to base regeneration on.
* @param {string} fieldValue - The value of the field to base regeneration on.
* @param {string} investigation - The regenerated investigation from Step 1 + Step 2 (YAML format).
* @param {string} investigationMetadata - The metadata/guide in YAML format.
* @param {string} [history] - Chat history in YAML format (optional, defaults to "-").
* @param {object} [contradictionInfo] - Optional contradiction information.
* @param {string | null} [contradictionInfo.contradictionReason] - Reason for contradiction.
* @param {string | null} [contradictionInfo.targetFieldName] - Target field name for contradiction.
* @param {boolean} [contradictionInfo.isContradicting] - Whether the field is contradicting.
* @returns {Promise<IAssistantInvestigationFormat | null>} - The regenerated steps only.
* @throws {Error} If the prompt is not found or regeneration fails.
*/
async regenerateOtherFieldsSteps(fieldName, fieldValue, investigation, investigationMetadata, history = "-", contradictionInfo) {
const logger = getLogger();
try {
logger.info(`Regenerating steps based on ${fieldName}`);
// Try to use dedicated prompt, fallback to existing prompt
let prompt = await Prompt.findOne({
name: "regenerate_other_fields_steps",
});
if (!prompt) {
prompt = await Prompt.findOne({
name: "generate_investigation_steps",
});
if (!prompt) {
throw new Error("generate_investigation_steps prompt not found.");
}
}
// Build prompt template - replace ALL occurrences of placeholders
let promptTemplate = prompt.template
.replaceAll("{guide}", investigationMetadata)
.replaceAll("{history}", history || "-")
.replaceAll("{fieldName}", fieldName)
.replaceAll("{fieldValue}", fieldValue);
// Only replace contradiction placeholders if there's actually a contradiction
if (contradictionInfo && contradictionInfo.isContradicting) {
promptTemplate = promptTemplate
.replaceAll("{contradictionReason}", contradictionInfo.contradictionReason || "-")
.replaceAll("{targetFieldName}", contradictionInfo.targetFieldName || "-")
.replaceAll("{isContradicting}", "true");
}
else {
// No contradiction - replace with empty/neutral values
promptTemplate = promptTemplate
.replaceAll("{contradictionReason}", "")
.replaceAll("{targetFieldName}", "")
.replaceAll("{isContradicting}", "false");
}
// If the prompt has {investigation} placeholder, replace ALL occurrences
if (promptTemplate.includes("{investigation}")) {
promptTemplate = promptTemplate.replaceAll("{investigation}", investigation);
}
try {
const parsedInvestigation = yamlModule.load(investigation);
const existingStepsCount = parsedInvestigation?.steps?.length || 0;
if (existingStepsCount > 0) {
const stepCountInstruction = `\n\n### CRITICAL REQUIREMENT:\nYou MUST return EXACTLY ${existingStepsCount} steps in your response. The investigation context shows ${existingStepsCount} existing steps, and you MUST regenerate and return ALL ${existingStepsCount} steps. Do NOT return fewer steps.\n`;
promptTemplate = promptTemplate + stepCountInstruction;
logger.info({
fieldName,
existingStepsCount,
}, "Added explicit instruction to return all steps");
}
}
catch (parseError) {
logger.warn({ fieldName, error: parseError }, "Failed to parse investigation YAML to count steps, continuing without step count instruction");
}
let retry = 0;
let response = null;
while (retry < 3) {
response = (await this.aiProcessor.fetchLLMResponse(promptTemplate, AssistantInvestigationFormat));
// Validate response has steps
if (response.steps === undefined || response.steps === null) {
retry += 1;
}
else {
break;
}
}
if (retry === 3 || !response) {
throw new Error(`Failed to regenerate steps based on ${fieldName}. Response is: ${JSON.stringify(response)}.`);
}
logger.info(`Response for regenerating steps is: ${JSON.stringify(response)}`);
return response;
}
catch (error) {
logger.error({
message: `Failed to regenerate steps based on ${fieldName}.`,
error: error instanceof Error ? error.message : "Unknown error",
});
throw error;
}
}
/**
* Regenerates other fields in an investigation based on a specific field.
* This operation uses a 3-step process: text fields → objects → steps.
* @category Services
* @param {string} investigationId - The ID of the investigation to update.
* @param {string} fieldName - The name of the field to base regeneration on (e.g., "title", "curriculum", "objectives").
* @param {Types.ObjectId | null} updatedBy - The user making the change (optional).
* @param {string} [history] - Chat history in YAML format (optional, defaults to "-").
* @returns {Promise<IInvestigation>} - The updated investigation.
* @throws {Error} If the investigation is not found or the field is invalid.
*/
async regenerateOtherFields(investigationId, fieldName, updatedBy = null, history = "-") {
const logger = getLogger();
try {
logger.info({ investigationId, fieldName }, "Regenerating other fields based on field");
// Get the investigation
const investigation = await Investigation.findById(investigationId);
if (!investigation) {
throw new Error("Investigation not found");
}
// Validate field name and get field value
const validBlockFields = [
"title",
"curriculum",
"unitNumberAndTitle",
"grade",
"lessonNumberAndTitle",
"objectives",
"ngss",
"analyticalFacts",
"goals",
"day",
];
const validStepFields = ["title", "descriptionEn", "desiredOutcome", "alternativeOutcome"];
const validObjectFields = ["name"];
let fieldValue;
let contradictionReason = null;
let targetFieldName = null;
let isContradicting = false;
// Check if it's a step field (format: steps.0.title, steps.1.descriptionEn, etc.)
const stepFieldMatch = fieldName.match(/^steps\.(\d+)\.(.+)$/);
if (stepFieldMatch) {
const stepIndexStr = stepFieldMatch[1];
const stepFieldName = stepFieldMatch[2];
if (!stepIndexStr || !stepFieldName) {
throw new Error(`Invalid step field format: ${fieldName}`);
}
const stepIndex = parseInt(stepIndexStr, 10);
if (!validStepFields.includes(stepFieldName)) {
throw new Error(`Invalid step field name: ${stepFieldName}. Valid step fields are: ${validStepFields.join(", ")}`);
}
const steps = investigation.steps || [];
if (stepIndex < 0 || stepIndex >= steps.length) {
throw new Error(`Step index ${stepIndex} is out of range. Investigation has ${steps.length} steps.`);
}
const step = steps[stepIndex];
if (!step) {
throw new Error(`Step at index ${stepIndex} not found`);
}
// Get the step field (step fields are stored as IEditableField in the model)
const stepModel = investigation.steps?.[stepIndex];
const stepField = stepModel?.[stepFieldName];
if (!stepField || !stepField.value) {
throw new Error(`Step field ${fieldName} has no value to base regeneration on`);
}
fieldValue = stepField.value;
contradictionReason = stepField.contradictionReason || null;
targetFieldName = stepField.targetFieldName || null;
isContradicting = stepField.isContradicting || false;
}
else if (fieldName === "name" || /^objects\.\d+\.name$/.test(fieldName)) {
// Object name field (supports legacy "name" or indexed "objects.{index}.name")
const objects = investigation.objects || [];
if (objects.length === 0) {
throw new Error("No objects found in investigation to base regeneration on");
}
if (fieldName === "name") {
// Legacy behavior: use first object
const firstObject = objects[0];
if (!firstObject || !firstObject.name || !firstObject.name.value) {
throw new Error("First object has no name to base regeneration on");
}
const objectNameField = firstObject.name;
fieldValue = objectNameField.value;
contradictionReason = objectNameField.contradictionReason || null;
targetFieldName = objectNameField.targetFieldName || null;
isContradicting = objectNameField.isContradicting || false;
}
else {
const match = fieldName.match(/^objects\.(\d+)\.(name)$/);
if (!match) {
throw new Error(`Invalid object field format: ${fieldName}. Expected objects.{index}.name`);
}
const objectIndex = parseInt(match[1], 10);
const objectFieldName = match[2];
if (!validObjectFields.includes(objectFieldName)) {
throw new Error(`Invalid object field name: ${objectFieldName}. Valid object fields are: ${validObjectFields.join(", ")}`);
}
if (objectIndex < 0 || objectIndex >= objects.length) {
throw new Error(`Object index ${objectIndex} is out of range. Investigation has ${objects.length} objects.`);
}
const object = objects[objectIndex];
const objectField = object?.[objectFieldName];
if (!objectField ||
typeof objectField !== "object" ||
!("value" in objectField) ||
!objectField.value) {
throw new Error(`Object field ${fieldName} has no value to base regeneration on`);
}
const editableField = objectField;
fieldValue = editableField.value;
contradictionReason = editableField.contradictionReason || null;
targetFieldName = editableField.targetFieldName || null;
isContradicting = editableField.isContradicting || false;
}
}
else {
// Block field
if (!validBlockFields.includes(fieldName)) {
throw new Error(`Invalid field name: ${fieldName}. Valid block fields are: ${validBlockFields.join(", ")}, valid step fields are: steps.{index}.{field}, valid object fields are: objects.{index}.name`);
}
// Get the field value and contradiction information
const field = investigation[fieldName];
if (!field || !field.value) {
throw new Error(`Field ${fieldName} has no value to base regeneration on`);
}
fieldValue = field.value;
contradictionReason = field.contradictionReason || null;
targetFieldName = field.targetFieldName || null;
isContradicting = field.isContradicting || false;
}
// Get metadata for the investigation
const unit = investigation.unitNumberAndTitle?.value?.split(":")[0]?.split(" ")[1];
const lesson = investigation.lessonNumberAndTitle?.value?.split(":")[0]?.split(" ")[1] || "0";
const lessonNumber = parseInt(lesson, 10);
const metadata = await Metadata.findOne({
unit: unit,
lessonNumber: Number.isNaN(lessonNumber) ? undefined : lessonNumber,
});
let investigationMetadataYaml = " ";
if (metadata) {
const yaml = yamlModule;
const investigationMetadataObject = metadata.toObject({
versionKey: false,
transform: (doc, ret) => {
delete ret._id;
return ret;
},
});
investigationMetadataYaml = yaml.dump(investigationMetadataObject);
}
// Convert current investigation to simple format for context
const currentInvestigation = convertInvestigationFromModel(investigation);
const yaml = yamlModule;
// Prepare contradiction info - only pass if field is actually contradicting
const contradictionInfo = isContradicting
? {
contradictionReason,
targetFieldName,
isContradicting,
}
: undefined;
const textFieldsOnly = {
title: currentInvestigation.title,
curriculum: currentInvestigation.curriculum,
unitNumberAndTitle: currentInvestigation.unitNumberAndTitle,
grade: currentInvestigation.grade,
lessonNumberAndTitle: currentInvestigation.lessonNumberAndTitle,
objectives: currentInvestigation.objectives,
ngss: currentInvestigation.ngss,
analyticalFacts: currentInvestigation.analyticalFacts,
goals: currentInvestigation.goals,
day: currentInvestigation.day,
};
logger.info({ investigationId, fieldName }, "Step 1: Regenerating text fields");
let regeneratedInvestigation = (await this.regenerateOtherFieldsWithoutStepsAndObjects(fieldName, fieldValue, yaml.dump(textFieldsOnly), investigationMetadataYaml, history || "-", contradictionInfo));
const investigationWithExistingObjects = {
...regeneratedInvestigation,
objects: currentInvestigation.objects || [],
};
logger.info({
investigationId,
fieldName,
existingObjectsCount: investigationWithExistingObjects.objects?.length || 0,
}, "Step 2: Regenerating objects (with existing objects in context)");
const regeneratedObjects = await this.regenerateOtherFieldsObjects(fieldName, fieldValue, yaml.dump(investigationWithExistingObjects), investigationMetadataYaml, history || "-", contradictionInfo);
regeneratedInvestigation.objects = regeneratedObjects?.objects;
const investigationWithExistingSteps = {
...regeneratedInvestigation,
steps: currentInvestigation.steps || [],
};
logger.info({
investigationId,
fieldName,
existingStepsCount: investigationWithExistingSteps.steps?.length || 0,
}, "Step 3: Regenerating steps (with existing steps in context)");
const regeneratedSteps = await this.regenerateOtherFieldsSteps(fieldName, fieldValue, yaml.dump(investigationWithExistingSteps), investigationMetadataYaml, history || "-", contradictionInfo);
regeneratedInvestigation.steps = regeneratedSteps?.steps;
logger.info({ investigationId, fieldName }, "All 3 steps completed. Building update DTO");
// Build update DTO - exclude the field we're basing regeneration on
const updateDto = {};
// Extract values from regenerated investigation
const responseData = regeneratedInvestigation;
logger.info({
investigationId,
fieldName,
responseDataKeys: Object.keys(responseData),
objectivesValue: responseData.objectives,
}, "Building update DTO from regenerated investigation");
const textFields = [
"title",
"curriculum",
"grade",
"unitNumberAndTitle",
"lessonNumberAndTitle",
"objectives",
"ngss",
"analyticalFacts",
"goals",
"day",
];
for (const field of textFields) {
// Explicitly skip if this field matches the fieldName used for regeneration
if (fieldName === field) {
logger.debug({ investigationId, fieldName, field }, "Skipping field in update DTO - matches regeneration source field");
continue;
}
const value = responseData[field];
if (value !== undefined && value !== null) {
updateDto[field] = value;
}
}
logger.info({
investigationId,
fieldName,
updateDtoKeys: Object.keys(updateDto),
hasObjectives: "objectives" in updateDto,
}, "Update DTO built");
let hasChanges = false;
const changedFields = [];
for (const field of textFields) {
const updateValue = updateDto[field];
if (updateValue !== undefined) {
const currentValue = investigation[field];
if (updateValue !== currentValue?.value) {
hasChanges = true;
changedFields.push(field);
}
}
}
if (updateDto.steps && Array.isArray(updateDto.steps) && updateDto.steps.length > 0) {
const existingSteps = investigation.steps || [];
// Check if fieldName is a step field (e.g., "steps.0.title")
const stepFieldMatch = fieldName.match(/^steps\.(\d+)\.(\w+)$/);
let stepIndexToSkip = null;
let stepFieldToSkip = null;
if (stepFieldMatch) {
stepIndexToSkip = parseInt(stepFieldMatch[1], 10);
stepFieldToSkip = stepFieldMatch[2];
}
logger.info({ stepFieldMatch, stepIndexToSkip, stepFieldToSkip });
// Check each step by index (steps format: [{ title?, descriptionEn?, ... }])
for (const [index, step] of updateDto.steps.entries()) {
const existingStep = existingSteps[index];
// Check each field in the step dynamically
for (const fieldKey in step) {
// Skip updating the step field that was used as regeneration source
if (stepIndexToSkip !== null &&
index === stepIndexToSkip &&
stepFieldToSkip === fieldKey) {
logger.debug({ investigationId, fieldName, stepIndex: index, stepField: fieldKey }, "Skipping step field update - matches regeneration source field");
continue;
}
const updateValue = step[fieldKey];
// Skip if field value is undefined or null
if (updateValue === undefined || updateValue === null) {
continue;
}
// Compare with existing value
const currentValue = existingStep?.[fieldKey]?.value;
if (updateValue !== currentValue) {
hasChanges = true;
const fieldPath = `steps[${index}].${fieldKey}`;
if (!changedFields.includes(fieldPath)) {
changedFields.push(fieldPath);
}
}
}
// If step is new (doesn't exist at this index), mark as changed
if (!existingStep) {
hasChanges = true;
changedFields.push(`steps[${index}]`);
}
}
}
if (updateDto.objects && Array.isArray(updateDto.objects) && updateDto.objects.length > 0) {
const existingObjects = investigation.objects || [];
// Check if fieldName is an object field (e.g., "objects.0.name" or legacy "name")
const match = fieldName.match(/^objects\.(\d+)\.name$/);
const objectIndexToSkip = fieldName === "name" ? 0 : match ? Number(match[1]) : null;
if (updateDto.objects.length !== existingObjects.length) {
hasChanges = true;
changedFields.push("objects");
}
else {
for (let i = 0; i < updateDto.objects.length; i++) {
// Skip updating the object that was used as regeneration source
if (objectIndexToSkip !== null && i === objectIndexToSkip) {
logger.debug({ investigationId, fieldName, objectIndex: i }, "Skipping object update - matches regeneration source field");
continue;
}
const regeneratedObject = updateDto.objects[i];
const existingObject = existingObjects[i];
if (regeneratedObject &&
existingObject &&
regeneratedObject.name !== existingObject.name?.value) {
hasChanges = true;
if (!changedFields.includes(`objects[${i}].name`)) {
changedFields.push(`objects[${i}].name`);
}
}
}
}
}
logger.info({
investigationId,
fieldName,
hasChanges,
changedFields,
totalChangedFields: changedFields.length,
}, "Comparison with current investigation completed");
if (responseData.steps &&
Array.isArray(responseData.steps) &&
responseData.steps.length > 0) {
const existingStepsCount = investigation.steps?.length || 0;
const regeneratedStepsCount = responseData.steps.length;
logger.info({
investigationId,
fieldName,
existingStepsCount,
regeneratedStepsCount,
allStepsRegenerated: regeneratedStepsCount >= existingStepsCount,
}, "Processing regenerated steps");
// Check if fieldName is a step field (e.g., "steps.0.title")
const stepFieldMatch = fieldName.match(/^steps\.(\d+)\.(\w+)$/);
let stepIndexToSkip = null;
let stepFieldToSkip = null;
if (stepFieldMatch) {
stepIndexToSkip = parseInt(stepFieldMatch[1], 10);
stepFieldToSkip = stepFieldMatch[2];
}
const mergedSteps = [];
const maxSteps = Math.max(regeneratedStepsCount, existingStepsCount);
for (let index = 0; index < maxSteps; index++) {
if (index < regeneratedStepsCount && responseData.steps[index]) {
const regeneratedStep = responseData.steps[index];
if (regeneratedStep) {
// If this step field matches the regeneration source, exclude that field
if (stepIndexToSkip !== null && index === stepIndexToSkip && stepFieldToSkip) {
const filteredStep = { ...regeneratedStep };
// Remove the field that matches the regeneration source
if (stepFieldToSkip in filteredStep) {
delete filteredStep[stepFieldToSkip];
logger.debug({ investigationId, fieldName, stepIndex: index, stepField: stepFieldToSkip }, "Excluding step field from merged steps - matches regeneration source field");
}
mergedSteps.push(filteredStep);
}
else {
mergedSteps.push(regeneratedStep);
}
}
}
else if (index < existingStepsCount) {
const existingStep = investigation.steps?.[index];
if (existingStep) {
mergedSteps.push({
title: existingStep.title?.value || "",
descriptionEn: existingStep.descriptionEn?.value || "",
desiredOutcome: existingStep.desiredOutcome?.value || "",
alternativeOutcome: existingStep.alternativeOutcome?.value || "",
skippable: existingStep.skippable?.value ?? null,
skippable_after_marking: existingStep.skippable_after_marking?.value ?? null,
});
}
}
}
if (regeneratedStepsCount < existingStepsCount) {
logger.warn({
investigationId,
fieldName,
existingStepsCount,
regeneratedStepsCount,
mergedStepsCount: mergedSteps.length,
}, "AI returned fewer steps than existing. Merged regenerated steps with existing steps.");
}
if (mergedSteps.length > 0) {
updateDto.steps = mergedSteps;
}
}
if (responseData.objects &&
Array.isArray(responseData.objects) &&
responseData.objects.length > 0) {
// Check if fieldName is an object field (e.g., "objects.0.name" or legacy "name")
const match = fieldName.match(/^objects\.(\d+)\.name$/);
const objectIndexToSkip = fieldName === "name" ? 0 : match ? Number(match[1]) : null;
const objectUpdates = [];
for (const [index, obj] of responseData.objects.entries()) {
if (!obj || typeof obj !== "object")
continue;
// Skip updating the object that was used as regeneration source
if (objectIndexToSkip !== null && index === objectIndexToSkip) {
const updateObj = investigation.objects?.[index];
logger.info({ updateObj }, "Update object ape jan");
if (updateObj) {
objectUpdates.push({
name: updateObj.name?.value
? { ...updateObj.name, value: updateObj.name.value }
: undefined,
objectId: updateObj.objectId?.value
? { ...updateObj.objectId, value: updateObj.objectId.value }
: undefined,
position: updateObj.position?.x?.value &&
updateObj.position?.y?.value &&
updateObj.position?.z?.value
? {
x: updateObj.position.x.value,
y: updateObj.position.y.value,
z: updateObj.position.z.value,
}
: undefined,
rotation: updateObj.rotation?.x?.value &&
updateObj.rotation?.y?.value &&
updateObj.rotation?.z?.value
? {
x: updateObj.rotation.x.value,
y: updateObj.rotation.y.value,
z: updateObj.rotation.z.value,
}
: undefined,
size: updateObj.size?.value ? updateObj.size.value : 1,
});
}
}
else {
const updateObj = {
name: obj.name ? { value: String(obj.name) } : undefined,
objectId: obj.objectId || obj.name
? { value: String(obj.objectId || obj.name || "") }
: undefined,
position: obj.position && typeof obj.position === "object"
? {
x: Number(obj.position.x) || 0,
y: Number(obj.position.y) || 0,
z: Number(obj.position.z) || 0,
}
: undefined,
rotation: obj.rotation && typeof obj.rotation === "object"
? {
x: Number(obj.rotation.x) || 0,
y: Number(obj.rotation.y) || 0,
z: Number(obj.rotation.z) || 0,
}
: undefined,
size: obj.size ? Number(obj.size) : 1,
};
objectUpdates.push(updateObj);
}
}
if (objectUpdates.length > 0) {
updateDto.objects = objectUpdates;
}
}
const oldInvestigation = JSON.parse(JSON.stringify(investigation));
const updateResult = await this.updateInvestigation(investigationId, updateDto, {
needsBuildUpdate: true,
updatedBy,
isAiGenerated: true, // Mark as AI-generated
skipVersioning: true, // We'll handle versioning via updateInvestigationByVersion
});
if (!updateResult.success || !updateResult.updatedInvestigation) {
throw new Error("Failed to update investigation with regenerated fields");
}
const updatedInvestigationDocument = await Investigation.findById(investigationId);
if (updatedInvestigationDocument) {
await this.updateInvestigationByVersion(oldInvestigation, updatedInvestigationDocument, updatedBy);
}
const updatedInvestigation = await this.getInvestigationById(investigationId);
updatedInvestigation._regenerationHasChanges = hasChanges;
updatedInvestigation._regenerationChangedFields = changedFields;
logger.info({ investigationId, fieldName, hasChanges, changedFieldsCount: changedFields.length }, "Successfully regenerated other fields based on field");
return updatedInvestigation;
}
catch (error) {
logger.error({
error,
investigationId,
fieldName,
message: error instanceof Error ? error.message : "Unknown error",
}, "Failed to regenerate other fields");
throw error;
}
}
}
Source