pch_xyz/ai
1
0
Fork 0
mirror of https://github.com/syuilo/ai.git synced 2025-03-25 21:12:56 +00:00
Code Issues Projects Releases Packages Wiki Activity

画像以外のファイルに対応&複数ファイルに対応など

Browse source 
- 定義を上部にまとめた
- 画像以外のファイルに対応
- 複数ファイルに対応
This commit is contained in:
tetsuya-ki 2025-01-13 13:32:45 +09:00
parent 1dbf7a2940
commit d19ba9562f
1 changed files with 73 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

112
src/modules/aichat/index.ts
View file

@ -15,10 +15,31 @@ type AiChat = {
key: string; key: string;
history?: { role: string; content: string }[]; history?: { role: string; content: string }[];
}; };
type Base64Image = { type base64File = {
type: string; type: string;
base64: string; base64: string;
url?: string;
}; };
type GeminiParts = {
inlineData?: {
mimeType: string;
data: string;
};
fileData?: {
mimeType: string;
fileUri: string;
};
text?: string;
}[];
type GeminiSystemInstruction = {
role: string;
parts: [{text: string}]
};
type GeminiContents = {
role: string;
parts: GeminiParts;
};
type AiChatHist = { type AiChatHist = {
postId: string; postId: string;
createdAt: number; createdAt: number;
@ -82,32 +103,34 @@ export default class extends Module {
} }
@bindThis @bindThis
private async genTextByGemini(aiChat: AiChat, image:Base64Image|null) { private async genTextByGemini(aiChat: AiChat, files:base64File[]) {
this.log('Generate Text By Gemini...'); this.log('Generate Text By Gemini...');
let parts: ({ text: string; inline_data?: undefined; } | { inline_data: { mime_type: string; data: string; }; text?: undefined; })[]; let parts: GeminiParts = [];
const systemInstruction : {role: string; parts: [{text: string}]} = {role: 'system', parts: [{text: aiChat.prompt}]}; const systemInstruction: GeminiSystemInstruction = {role: 'system', parts: [{text: aiChat.prompt}]};
if (!image) { parts = [{text: aiChat.question}];
// 画像がない場合、メッセージのみで問い合わせ // ファイルが存在する場合、画像を添付して問い合わせ
parts = [{text: aiChat.question}]; if (files.length >= 1) {
} else { for (const file of files){
// 画像が存在する場合、画像を添付して問い合わせ parts.push(
parts = [ {
{ text: aiChat.question }, inlineData: {
{ mimeType: file.type,
inline_data: { data: file.base64,
mime_type: image.type, },
data: image.base64, }
}, );
}, }
];
} }
// 履歴を追加 // 履歴を追加
let contents: ({ role: string; parts: ({ text: string; inline_data?: undefined; } | { inline_data: { mime_type: string; data: string; }; text?: undefined; })[]}[]) = []; let contents: GeminiContents[] = [];
if (aiChat.history != null) { if (aiChat.history != null) {
aiChat.history.forEach(entry => { aiChat.history.forEach(entry => {
contents.push({ role : entry.role, parts: [{text: entry.content}]}); contents.push({
role : entry.role,
parts: [{text: entry.content}],
});
}); });
} }
contents.push({role: 'user', parts: parts}); contents.push({role: 'user', parts: parts});
@ -193,31 +216,42 @@ export default class extends Module {
} }
@bindThis @bindThis
private async note2base64Image(notesId: string) { private async note2base64File(notesId: string) {
const noteData = await this.ai.api('notes/show', { noteId: notesId }); const noteData = await this.ai.api('notes/show', { noteId: notesId });
let fileType: string | undefined,thumbnailUrl: string | undefined; let files:base64File[] = [];
let fileType: string | undefined, filelUrl: string | undefined;
if (noteData !== null && noteData.hasOwnProperty('files')) { if (noteData !== null && noteData.hasOwnProperty('files')) {
if (noteData.files.length > 0) { for (let i = 0; i < noteData.files.length; i++) {
if (noteData.files[0].hasOwnProperty('type')) { if (noteData.files[i].hasOwnProperty('type')) {
fileType = noteData.files[0].type; fileType = noteData.files[i].type;
if (noteData.files[i].hasOwnProperty('name')) {
// 拡張子で挙動を変えようと思ったが、text/plainしかMisskeyで変になってGemini対応してるものがない？
// let extention = noteData.files[i].name.split('.').pop();
if (fileType === 'application/octet-stream' || fileType === 'application/xml') {
fileType = 'text/plain';
}
}
} }
if (noteData.files[0].hasOwnProperty('thumbnailUrl')) { if (noteData.files[i].hasOwnProperty('thumbnailUrl') && noteData.files[i].thumbnailUrl) {
thumbnailUrl = noteData.files[0].thumbnailUrl; filelUrl = noteData.files[i].thumbnailUrl;
} else if (noteData.files[i].hasOwnProperty('url') && noteData.files[i].url) {
filelUrl = noteData.files[i].url;
} }
} if (fileType !== undefined && filelUrl !== undefined) {
if (fileType !== undefined && thumbnailUrl !== undefined) { try {
try { this.log('filelUrl:'+filelUrl);
const image = await urlToBase64(thumbnailUrl); const file = await urlToBase64(filelUrl);
const base64Image:Base64Image = {type: fileType, base64: image}; const base64file:base64File = {type: fileType, base64: file};
return base64Image; files.push(base64file);
} catch (err: unknown) { } catch (err: unknown) {
if (err instanceof Error) { if (err instanceof Error) {
this.log(`${err.name}\n${err.message}\n${err.stack}`); this.log(`${err.name}\n${err.message}\n${err.stack}`);
}
} }
} }
} }
} }
return null; return files;
} }
@bindThis @bindThis
@ -424,7 +458,7 @@ export default class extends Module {
msg.reply(serifs.aichat.nothing(exist.type)); msg.reply(serifs.aichat.nothing(exist.type));
return false; return false;
} }
const base64Image: Base64Image | null = await this.note2base64Image(msg.id); const base64Files: base64File[] = await this.note2base64File(msg.id);
aiChat = { aiChat = {
question: question, question: question,
prompt: prompt, prompt: prompt,
@ -435,7 +469,7 @@ export default class extends Module {
if (exist.api) { if (exist.api) {
aiChat.api = exist.api aiChat.api = exist.api
} }
text = await this.genTextByGemini(aiChat, base64Image); text = await this.genTextByGemini(aiChat, base64Files);
break; break;
case TYPE_PLAMO: case TYPE_PLAMO: