fix import (#242)

* 🐛 fix import (#224)

* 🐛 fix import of corrupted history

see https://github.com/mckaywrigley/chatbot-ui/issues/224#issuecomment-1486080888

* add the run-test-suite github action

.dockerignore

@@ -1,4 +1,4 @@
 .env
 .env.local
 node_modules
-
+test-results

.github/workflows/run-test-suite.yml

@@ -0,0 +1,31 @@
+name: Run Jest Tests
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    container:
+      image: node:16
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v2
+
+    - name: Install dependencies
+      run: npm ci
+
+    - name: Run Jest Test Suite
+      run: npm test
+    
+    - name: Publish Test Report
+      if: always()
+      uses: EnricoMi/publish-unit-test-result-action@v1
+      with:
+        files: test-results/**/results.xml

.gitignore

@@ -7,6 +7,7 @@
 
 # testing
 /coverage
+/test-results
 
 # next.js
 /.next/

__tests__/utils/app/importExports.test.ts

@@ -0,0 +1,156 @@
+import { ExportFormatV1, ExportFormatV2 } from '@/types/export';
+import { OpenAIModels, OpenAIModelID } from '@/types/openai';
+import { DEFAULT_SYSTEM_PROMPT } from '@/utils/app/const';
+import {
+  cleanData,
+  isExportFormatV1,
+  isExportFormatV2,
+  isExportFormatV3,
+  isLatestExportFormat,
+} from '@/utils/app/importExport';
+
+describe('Export Format Functions', () => {
+  describe('isExportFormatV1', () => {
+    it('should return true for v1 format', () => {
+      const obj = [{ id: 1 }];
+      expect(isExportFormatV1(obj)).toBe(true);
+    });
+
+    it('should return false for non-v1 formats', () => {
+      const obj = { version: 3, history: [], folders: [] };
+      expect(isExportFormatV1(obj)).toBe(false);
+    });
+  });
+
+  describe('isExportFormatV2', () => {
+    it('should return true for v2 format', () => {
+      const obj = { history: [], folders: [] };
+      expect(isExportFormatV2(obj)).toBe(true);
+    });
+
+    it('should return false for non-v2 formats', () => {
+      const obj = { version: 3, history: [], folders: [] };
+      expect(isExportFormatV2(obj)).toBe(false);
+    });
+  });
+
+  describe('isExportFormatV3', () => {
+    it('should return true for v3 format', () => {
+      const obj = { version: 3, history: [], folders: [] };
+      expect(isExportFormatV3(obj)).toBe(true);
+    });
+
+    it('should return false for non-v3 formats', () => {
+      const obj = { version: 4, history: [], folders: [] };
+      expect(isExportFormatV3(obj)).toBe(false);
+    });
+  });
+});
+
+describe('cleanData Functions', () => {
+  describe('cleaning v1 data', () => {
+    it('should return the latest format', () => {
+      const data = [
+        {
+          id: 1,
+          name: 'conversation 1',
+          messages: [
+            {
+              role: 'user',
+              content: "what's up ?",
+            },
+            {
+              role: 'assistant',
+              content: 'Hi',
+            },
+          ],
+        },
+      ] as ExportFormatV1;
+      const obj = cleanData(data);
+      expect(isLatestExportFormat(obj)).toBe(true);
+      expect(obj).toEqual({
+        version: 3,
+        history: [
+          {
+            id: 1,
+            name: 'conversation 1',
+            messages: [
+              {
+                role: 'user',
+                content: "what's up ?",
+              },
+              {
+                role: 'assistant',
+                content: 'Hi',
+              },
+            ],
+            model: OpenAIModels[OpenAIModelID.GPT_3_5],
+            prompt: DEFAULT_SYSTEM_PROMPT,
+            folderId: null,
+          },
+        ],
+        folders: [],
+      });
+    });
+  });
+
+  describe('cleaning v2 data', () => {
+    it('should return the latest format', () => {
+      const data = {
+        history: [
+          {
+            id: '1',
+            name: 'conversation 1',
+            messages: [
+              {
+                role: 'user',
+                content: "what's up ?",
+              },
+              {
+                role: 'assistant',
+                content: 'Hi',
+              },
+            ],
+          },
+        ],
+        folders: [
+          {
+            id: 1,
+            name: 'folder 1',
+          },
+        ],
+      } as ExportFormatV2;
+      const obj = cleanData(data);
+      expect(isLatestExportFormat(obj)).toBe(true);
+      expect(obj).toEqual({
+        version: 3,
+        history: [
+          {
+            id: '1',
+            name: 'conversation 1',
+            messages: [
+              {
+                role: 'user',
+                content: "what's up ?",
+              },
+              {
+                role: 'assistant',
+                content: 'Hi',
+              },
+            ],
+            model: OpenAIModels[OpenAIModelID.GPT_3_5],
+            prompt: DEFAULT_SYSTEM_PROMPT,
+            folderId: null,
+          },
+        ],
+        folders: [
+          {
+            id: '1',
+            name: 'folder 1',
+            type: 'chat',
+          },
+        ],
+      });
+    });
+  });
+});

components/Chatbar/Chatbar.tsx

@@ -1,5 +1,6 @@
 import { Conversation } from '@/types/chat';
 import { KeyValuePair } from '@/types/data';
+import { SupportedExportFormats } from '@/types/export';
 import { Folder } from '@/types/folder';
 import {
   IconArrowBarLeft,
@@ -36,10 +37,7 @@ interface Props {
   onApiKeyChange: (apiKey: string) => void;
   onClearConversations: () => void;
   onExportConversations: () => void;
-  onImportConversations: (data: {
-    conversations: Conversation[];
-    folders: Folder[];
-  }) => void;
+  onImportConversations: (data: SupportedExportFormats) => void;
 }
 
 export const Chatbar: FC<Props> = ({

components/Chatbar/ChatbarSettings.tsx

@@ -1,5 +1,4 @@
-import { Conversation } from '@/types/chat';
-import { Folder } from '@/types/folder';
+import { SupportedExportFormats } from '@/types/export';
 import { IconFileExport, IconMoon, IconSun } from '@tabler/icons-react';
 import { useTranslation } from 'next-i18next';
 import { FC } from 'react';
@@ -16,10 +15,7 @@ interface Props {
   onApiKeyChange: (apiKey: string) => void;
   onClearConversations: () => void;
   onExportConversations: () => void;
-  onImportConversations: (data: {
-    conversations: Conversation[];
-    folders: Folder[];
-  }) => void;
+  onImportConversations: (data: SupportedExportFormats) => void;
 }
 
 export const ChatbarSettings: FC<Props> = ({

components/Settings/Import.tsx

@@ -1,16 +1,11 @@
-import { Conversation } from '@/types/chat';
-import { Folder } from '@/types/folder';
-import { cleanConversationHistory } from '@/utils/app/clean';
+import { SupportedExportFormats } from '@/types/export';
 import { IconFileImport } from '@tabler/icons-react';
 import { useTranslation } from 'next-i18next';
 import { FC } from 'react';
 import { SidebarButton } from '../Sidebar/SidebarButton';
 
 interface Props {
-  onImport: (data: {
-    conversations: Conversation[];
-    folders: Folder[];
-  }) => void;
+  onImport: (data: SupportedExportFormats) => void;
 }
 
 export const Import: FC<Props> = ({ onImport }) => {
@@ -30,12 +25,7 @@ export const Import: FC<Props> = ({ onImport }) => {
           const reader = new FileReader();
           reader.onload = (e) => {
             let json = JSON.parse(e.target?.result as string);
-
-            if (json && !json.folders) {
-              json = { history: cleanConversationHistory(json), folders: [] };
-            }
-
-            onImport({ conversations: json.history, folders: json.folders });
+            onImport(json);
           };
           reader.readAsText(file);
         }}

jest.config.ts

@@ -0,0 +1,22 @@
+import type { Config } from 'jest';
+
+const config: Config = {
+  testEnvironment: 'jest-environment-jsdom',
+  verbose: true,
+  preset: 'ts-jest',
+  moduleNameMapper: {
+    '^@/(.*)$': '<rootDir>/$1',
+  },
+  reporters: [
+    'default',
+    [
+      'jest-junit',
+      {
+        outputDirectory: 'test-results/jest',
+        outputName: 'results.xml',
+      },
+    ],
+  ],
+};
+
+export default config;

package.json

@@ -7,7 +7,8 @@
     "build": "next build",
     "start": "next start",
     "lint": "next lint",
-    "format": "prettier --write ."
+    "format": "prettier --write .",
+    "test": "jest"
   },
   "dependencies": {
     "@dqbd/tiktoken": "^1.0.2",
@@ -29,6 +30,9 @@
   },
   "devDependencies": {
     "@tailwindcss/typography": "^0.5.9",
+    "@testing-library/jest-dom": "^5.16.5",
+    "@testing-library/react": "^14.0.0",
+    "@types/jest": "^29.5.0",
     "@types/node": "18.15.0",
     "@types/react": "18.0.28",
     "@types/react-dom": "18.0.11",
@@ -37,10 +41,15 @@
     "autoprefixer": "^10.4.14",
     "eslint": "8.36.0",
     "eslint-config-next": "13.2.4",
+    "jest": "^29.5.0",
+    "jest-environment-jsdom": "^29.5.0",
+    "jest-junit": "^15.0.0",
     "postcss": "^8.4.21",
     "prettier": "^2.8.7",
     "prettier-plugin-tailwindcss": "^0.2.5",
     "tailwindcss": "^3.2.7",
+    "ts-jest": "^29.0.5",
+    "ts-node": "^10.9.1",
     "typescript": "4.9.5"
   }
 }

pages/index.tsx

@@ -5,6 +5,7 @@ import { Promptbar } from '@/components/Promptbar/Promptbar';
 import { ChatBody, Conversation, Message } from '@/types/chat';
 import { KeyValuePair } from '@/types/data';
 import { ErrorMessage } from '@/types/error';
+import { LatestExportFormat, SupportedExportFormats } from '@/types/export';
 import { Folder, FolderType } from '@/types/folder';
 import { OpenAIModel, OpenAIModelID, OpenAIModels } from '@/types/openai';
 import { Prompt } from '@/types/prompt';
@@ -285,19 +286,12 @@ const Home: React.FC<HomeProps> = ({ serverSideApiKeyIsSet }) => {
     exportData();
   };
 
-  const handleImportConversations = (data: {
-    conversations: Conversation[];
-    folders: Folder[];
-  }) => {
-    const updatedConversations = [...conversations, ...data.conversations];
-    const updatedFolders = [...folders, ...data.folders];
+  const handleImportConversations = (data: SupportedExportFormats) => {
+    const { history, folders }: LatestExportFormat = importData(data);
 
-    importData(updatedConversations, updatedFolders);
-    setConversations(updatedConversations);
-    setSelectedConversation(
-      updatedConversations[updatedConversations.length - 1],
-    );
-    setFolders(updatedFolders);
+    setConversations(history);
+    setSelectedConversation(history[history.length - 1]);
+    setFolders(folders);
   };
 
   const handleSelectConversation = (conversation: Conversation) => {

tsconfig.json

@@ -1,7 +1,11 @@
 {
   "compilerOptions": {
     "target": "es5",
-    "lib": ["dom", "dom.iterable", "esnext"],
+    "lib": [
+      "dom",
+      "dom.iterable",
+      "esnext"
+    ],
     "allowJs": true,
     "skipLibCheck": true,
     "strict": true,
@@ -15,9 +19,21 @@
     "jsx": "preserve",
     "incremental": true,
     "paths": {
-      "@/*": ["./*"]
+      "@/*": [
+        "./*"
+      ]
     }
   },
-  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx"],
-  "exclude": ["node_modules"]
-}
+  "include": [
+    "next-env.d.ts",
+    "**/*.ts",
+    "**/*.tsx"
+  ],
+  "exclude": [
+    "node_modules"
+  ],
+  "jest": {
+    "preset": "ts-jest",
+    "testEnvironment": "jsdom"
+  }
+}

types/export.ts

@@ -0,0 +1,36 @@
+import { Conversation, Message } from './chat';
+import { Folder } from './folder';
+import { OpenAIModel } from './openai';
+
+export type SupportedExportFormats =
+  | ExportFormatV1
+  | ExportFormatV2
+  | ExportFormatV3;
+export type LatestExportFormat = ExportFormatV3;
+
+////////////////////////////////////////////////////////////////////////////////////////////
+interface ConversationV1 {
+  id: number;
+  name: string;
+  messages: Message[];
+}
+
+export type ExportFormatV1 = ConversationV1[];
+
+////////////////////////////////////////////////////////////////////////////////////////////
+interface ChatFolder {
+  id: number;
+  name: string;
+}
+
+export interface ExportFormatV2 {
+  history: Conversation[] | null;
+  folders: ChatFolder[] | null;
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////
+export interface ExportFormatV3 {
+  version: 3;
+  history: Conversation[];
+  folders: Folder[];
+}

utils/app/clean.ts

@@ -36,13 +36,18 @@ export const cleanSelectedConversation = (conversation: Conversation) => {
   return updatedConversation;
 };
 
-export const cleanConversationHistory = (history: Conversation[]) => {
+export const cleanConversationHistory = (history: any[]): Conversation[] => {
   // added model for each conversation (3/20/23)
   // added system prompt for each conversation (3/21/23)
   // added folders (3/23/23)
   // added prompts (3/26/23)
 
-  return history.reduce((acc: Conversation[], conversation) => {
+  if (!Array.isArray(history)) {
+    console.warn('history is not an array. Returning an empty array.');
+    return [];
+  }
+
+  return history.reduce((acc: any[], conversation) => {
     try {
       if (!conversation.model) {
         conversation.model = OpenAIModels[OpenAIModelID.GPT_3_5];

utils/app/importExport.ts

@@ -1,5 +1,53 @@
-import { Conversation } from '@/types/chat';
-import { Folder } from '@/types/folder';
+import {
+  ExportFormatV1,
+  ExportFormatV2,
+  ExportFormatV3,
+  LatestExportFormat,
+  SupportedExportFormats,
+} from '@/types/export';
+import { cleanConversationHistory } from './clean';
+
+export function isExportFormatV1(obj: any): obj is ExportFormatV1 {
+  return Array.isArray(obj);
+}
+
+export function isExportFormatV2(obj: any): obj is ExportFormatV2 {
+  return !('version' in obj) && 'folders' in obj && 'history' in obj;
+}
+
+export function isExportFormatV3(obj: any): obj is ExportFormatV3 {
+  return obj.version === 3;
+}
+
+export const isLatestExportFormat = isExportFormatV3;
+
+export function cleanData(data: SupportedExportFormats): LatestExportFormat {
+  if (isExportFormatV1(data)) {
+    return {
+      version: 3,
+      history: cleanConversationHistory(data),
+      folders: [],
+    };
+  }
+
+  if (isExportFormatV2(data)) {
+    return {
+      version: 3,
+      history: cleanConversationHistory(data.history || []),
+      folders: (data.folders || []).map((chatFolder) => ({
+        id: chatFolder.id.toString(),
+        name: chatFolder.name,
+        type: 'chat',
+      })),
+    };
+  }
+
+  if (isExportFormatV3(data)) {
+    return data;
+  }
+
+  throw new Error('Unsupported data format');
+}
 
 function currentDate() {
   const date = new Date();
@@ -21,9 +69,10 @@ export const exportData = () => {
   }
 
   const data = {
-    history,
-    folders,
-  };
+    version: 3,
+    history: history || [],
+    folders: folders || [],
+  } as LatestExportFormat;
 
   const blob = new Blob([JSON.stringify(data, null, 2)], {
     type: 'application/json',
@@ -40,13 +89,18 @@ export const exportData = () => {
 };
 
 export const importData = (
-  conversations: Conversation[],
-  folders: Folder[],
-) => {
+  data: SupportedExportFormats,
+): LatestExportFormat => {
+  const cleanedData = cleanData(data);
+
+  const conversations = cleanedData.history;
   localStorage.setItem('conversationHistory', JSON.stringify(conversations));
   localStorage.setItem(
     'selectedConversation',
     JSON.stringify(conversations[conversations.length - 1]),
   );
-  localStorage.setItem('folders', JSON.stringify(folders));
+
+  localStorage.setItem('folders', JSON.stringify(cleanedData.folders));
+
+  return cleanedData;
 };