From 06be9537e423de49375ee7dc14f1b1dbdae6d574 Mon Sep 17 00:00:00 2001 From: Souryabrata <123974482+BrataBuilds@users.noreply.github.com> Date: Tue, 7 Oct 2025 06:49:38 +0530 Subject: [PATCH 1/3] update spelling --- CONTRIBUTING.md | 88 +++++++++++++++++++++++++++++-------------------- 1 file changed, 53 insertions(+), 35 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4f3628b..09c8faa 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,110 +1,128 @@ # CONTRIBUTING + This document contains the list of issues, suggestions and improvements that can be added to this project and a proper guide on how to contribute to the project. + - [CONTRIBUTING](#contributing) - [Registration](#registration) - - [Guildlines :](#guildlines-) - - [List of major issues :](#list-of-major-issues-) + - [Guidelines](#guidelines) + - [List of major issues](#list-of-major-issues) + - [Related to UI :-](#related-to-ui--) + - [Frontend+Backend :-](#frontendbackend--) + - [Backend :-](#backend--) - [List of Improvements and Suggestions](#list-of-improvements-and-suggestions) - - [Improvments to the UI](#improvments-to-the-ui) - - [Long-term Goals :](#long-term-goals-) + - [Improvements to the UI](#improvements-to-the-ui) + - [Long-term Goals](#long-term-goals) - [Improvements to Backend :-](#improvements-to-backend--) - - [Long term Goals :](#long-term-goals--1) - - [In game control :](#in-game-control-) - - [Cross platform compatibility :](#cross-platform-compatibility-) + - [Long term Goals](#long-term-goals-1) + - [In game control](#in-game-control) + - [Cross platform compatibility](#cross-platform-compatibility) - [Other Known Bugs and Issues : ๐Ÿชฒ](#other-known-bugs-and-issues--) -## Registration +## Registration +> >[!IMPORTANT] -> All contributors must do the following to be eligible for MLSA X HACKTOBERFEST: +> All contributors must do the following to be eligible for MLSA X HACKTOBERFEST: + - [Star this repository](https://github.com/keploy/keploy) - [Register here on MLSA KIIT website](https://register.mlsakiit.com/) - [Register on HacktoberFest Official Website](https://hacktoberfest.com/auth/) -## Guildlines : -Adhere to [Hacktober Fest Guidlines](https://hacktoberfest.com/) and maintain common ettiquette of contributing to an open source project. +## Guidelines -If you have any questions regarding contributing to this repository please contact the contributor. +Adhere to [Hacktober Fest Guidelines](https://hacktoberfest.com/) and maintain common etiquette of contributing to an open source project. + +If you have any questions regarding contributing to this repository please contact the contributor. -### List of major issues : +### List of major issues +> >[!IMPORTANT] > It is highly recommended for contributors to first have a look at the list of major issues work on them with higher priority. -**Related to UI :-** +#### Related to UI :- + 1. Make the window resizable and all window elements scalable. 2. Add window control buttons for minimise and fullscreen (to be added after the previous implementation) -3. Whenever you hover over the screenshot button, placehoder text is inserted in the chatbox but it doesn't go away after you stop hovering, and you have to manually press backspace to remove the text. +3. Whenever you hover over the screenshot button, placeholder text is inserted in the chat box but it doesn't go away after you stop hovering, and you have to manually press backspace to remove the text. 4. Refactor the codebase of the overlay modular so that it is easier to work with. -**Frontend+Backend :-** +#### Frontend+Backend :- + 1. Chat history is not stored, if you try to follow up gemini with what you asked in the previous chat it will have 0 idea what you are talking about. >[!TIP] -> Here is a suggested solution: -Store the chats of the user in a database, (we are already using sqlite for screenshots, might as well use it), then when we give a new prompt to gemini, old chats are added to the prompt. +> Here is a suggested solution: +Store the chats of the user in a database, (we are already using sqlite for screenshots, might as well use it), then when we give a new prompt to gemini, old chats are added to the prompt. + - Only store the last 30 chats or so. - Every game will have its own database table, i.e. chats are stored on a per game basis. - Additional meta-data such as timestamp should also be stored so when user asked "What did I do yesterday?" it should be able to retrieve the screenshot from 24 hours ago etc. - >[!IMPORTANT] > UI/UX Addition : + - Add the ability to view the stored chats across various sessions/games from the overlay. -- Add the configuration in the settings menu to read, delete and edit them. +- Add the configuration in the settings menu to read, delete and edit them. - Add a setting to set how many chats per game/session to store. -**Backend :-** +#### Backend :- + 1. Chroma db vector collections aren't searched properly, this may have to do with the chroma client not being initialised properly or the collections are not being created properly in `get_or_create_collection()` or the incorrect implementation of `search_knowledge()` in *vector_service.py*. This issue requires a more thorough investigation. 2. Web Scrapper in *knowledge_manager.py* sometimes gets blocked by certain websites, (*namely* the ones present in *minecraft.csv*) + >[!TIP] > Recommended Solutions : + - Use Proxies to circumvent IP bans. - Rotate a list of User Agents and headers. - Make it asynchronous using `asyncio + httpx` - - - ## List of Improvements and Suggestions +> >[!TIP] > Feel free to give us any of your ideas, suggestions and feedback to add to this list. -### Improvments to the UI -1. Add Markdown support for Gemini's reponses. +### Improvements to the UI + +1. Add Markdown support for Gemini's responses. 2. Improve the chat Window UI such that it is easier to tell apart the messages of the user and Pixly. 3. Add the ability to view images results from the web and play recommended youtube videos directly within the overlay. +#### Long-term Goals - -#### Long-term Goals : 1. Add the ability for users to add custom themes of the overlay, different themes belonging to different games, so when the overlay can automatically apply a certain theme when a particular game is detected. ### Improvements to Backend :- 1. Add more .csv entries about wikis, guides, youtube videos, forum posts about more games, especially single player story based titles like `Elden Ring, Hollow Knight : Silksong, Black Myth: Wukong,Cyberpunk 2077` 2. Implement a Better way to store screenshots: + >[!TIP] -> Suggested Improvemments : +> Suggested Improvements : + - Vectorise the screenshots as well. - Add tool calling for the agent to call a tool to retrieve the screenshot from a specific time or from a specific game. + 1. Improve the Web Scrapper : In Addition to fixing the above issues, add the ability to scrape youtube audio transcriptions. +### Long term Goals + +#### In game control -### Long term Goals : -#### In game control : Add the ability for the agent to control the game and play the game for you and perform repetitive tasks, such as : `Build a small house for me in minecraft.` or `Automatically plant and regrow my crops while I afk` -#### Cross platform compatibility : +#### Cross platform compatibility + Reliance on the win32 api for taking screenshots means we can't transition to a different platform, and are stuck with Windows for now. In the future we may wanna add cross platform compatibility with Linux. ## Other Known Bugs and Issues : ๐Ÿชฒ +> > [!IMPORTANT] -> These are a bit obscure and their causes aren't known yet. +> These are a bit obscure and their causes aren't known yet. + 1. Overlay hangs and then crashes when turning off the *enable screenshots setting.* 2. `game_detection.py`, it reports the incorrect game being detected, in some cases. 3. After adding your API Key from the overlay, sometimes it shows that the user has added their key, sometimes it doesn't. - - From 6a747ef90e51a0ea44aa0011e93ff57a14bea62a Mon Sep 17 00:00:00 2001 From: Souryabrata <123974482+BrataBuilds@users.noreply.github.com> Date: Tue, 7 Oct 2025 06:49:53 +0530 Subject: [PATCH 2/3] update spelling --- README.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e90c469..cbae552 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,6 @@ Pixly - Your AI Gaming Assistant ๐ŸŽฎ - Pixly is a desktop overlay that acts as your gaming assistant, combining AI chat with automated, privacy-friendly screenshot capture and a game-specific Retrieval-Augmented Generation (RAG) knowledge base. Pixly detects what game you're playing, retrieves relevant, curated knowledge (wikis, user-supplied YouTube descriptions, and forum posts) via a local vector database, and grounds Gemini responses on those sources. Make sure to star our repository, your support is much appreciated. @@ -19,6 +18,7 @@ Make sure to star our repository, your support is much appreciated. >[!IMPORTANT] > ๐ŸŽƒ Hacktoberfest 2025 Participant > Please make sure to [star this repo](https://github.com/keploy/keploy). +> ## ๐Ÿ“‹ Table of Contents - [๐Ÿ“‹ Table of Contents](#-table-of-contents) @@ -45,7 +45,6 @@ Make sure to star our repository, your support is much appreciated. - ๐Ÿ“– For Contributing Visit [CONTRIBUTING.md](https://github.com/MLSAKIIT/pixly/blob/main/CONTRIBUTING.md) - โš™๏ธ For Setup and Installation visit [INSTALL.md](https://github.com/MLSAKIIT/pixly/blob/main/INSTALL.md) - ## ๐ŸŽฎ What Pixly Does - ๐Ÿค– Intelligent, game-focused chat using Google Gemini with a "Game Expert" system prompt @@ -54,18 +53,19 @@ Make sure to star our repository, your support is much appreciated. - ๐Ÿ” RAG pipeline over per-game CSV knowledge with local vector search (Chroma) - ๐Ÿ’ป Modern desktop overlay for chatting, settings, and screenshot gallery - ## ๐Ÿ—๏ธ Architecture Overview Pixly is organized into three main layers: UI Overlay, Backend API, and AI/RAG services, all running locally. ### 1) UI Overlay (`overlay.py`) + - CustomTkinter-based floating overlay, always-on-top, draggable - Chat window with typing indicator and styled messages (user vs assistant) - Settings window to manage screenshot capture and set the Google API key (persisted to `.env` via backend) - Screenshot gallery with View and Delete actions ### 2) Backend API (`backend/`) + - FastAPI server exposes HTTP endpoints on 127.0.0.1:8000 - Responsibilities: - Route chat requests to Gemini @@ -76,6 +76,7 @@ Pixly is organized into three main layers: UI Overlay, Backend API, and AI/RAG s - Manage API key configuration (.env persistence + live reconfigure) Key modules: + - `backend/backend.py`: API endpoints and routing - `backend/chatbot.py`: Gemini client configuration, runtime reconfigure, and chat logic (with RAG context injection) - `backend/screenshot.py`: Encrypted screenshot capture and storage; database operations; delete support @@ -84,6 +85,7 @@ Key modules: - `backend/vector_service.py`: Chroma persistent client, collection management, chunking, embeddings, and semantic queries ### 3) AI & RAG Layer + - Model: `Google Gemini 2.5 Flash Lite` for responses - System prompt (`PROMPTS.txt`) defines "Game Expert" persona and instructs grounding answers in retrieved snippets (WIKI / YOUTUBE / FORUM) with URLs - Vector DB: `Chroma` (persistent on disk in `vector_db/`) @@ -106,6 +108,7 @@ wiki,wiki_desc,youtube,yt_desc,forum,forum_desc - **forum_desc**: Contributor-provided description of the forum URL Processing pipeline per game: + 1. Load CSV for the game (e.g., `games_info/minecraft.csv`) 2. Extract text from wiki and forum URLs; keep YouTube descriptions as-is 3. Clean and chunk text into manageable segments (e.g., ~512 tokens) @@ -117,6 +120,7 @@ Vector DB collections are organized by game and source type, e.g. `minecraft_wik ## ๐ŸŽฏ Game Detection Pixly uses a layered strategy to infer the current game: + - **Process Detection**: Scans running processes for known executables - **Screenshot Context**: Uses recent screenshot metadata (app/window) when available - **Manual Override**: Detects game mentions in the user's message (e.g., "I'm playing Minecraft") @@ -183,6 +187,7 @@ pixly/ - **System**: psutil + pywin32 for Windows process/window info; Pillow for imaging Notes: + - The embedding model is configurable; by default we use a sentence-transformers model suitable for local inference. The system can be switched to a different embedder (e.g., Mistral embeddings) with minor changes in `vector_service.py`. - The persona and grounding behavior are controlled by `PROMPTS.txt` so Gemini cites sources from retrieved snippets and focuses answers on gaming topics. From d013a09cac725269a69e84ab71c950c64971107d Mon Sep 17 00:00:00 2001 From: Souryabrata <123974482+BrataBuilds@users.noreply.github.com> Date: Mon, 13 Oct 2025 13:04:19 +0530 Subject: [PATCH 3/3] Fixed Collections not found Error Updated Contributing.md --- .gitignore | 3 ++- CONTRIBUTING.md | 42 ++++++++++++++++++++++---------------- games_info/minecraft.csv | 8 +++----- services/chatbot.py | 6 +++--- services/vector_service.py | 6 +++--- test_system.py | 2 +- test_system_BMW.py | 1 - 7 files changed, 36 insertions(+), 32 deletions(-) diff --git a/.gitignore b/.gitignore index 859ac18..0e92542 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,5 @@ wheels/ *.json *.db vector_db/ -.coverage \ No newline at end of file +.coverage +.vscode/ \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 09c8faa..e258595 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -31,9 +31,8 @@ This document contains the list of issues, suggestions and improvements that can Adhere to [Hacktober Fest Guidelines](https://hacktoberfest.com/) and maintain common etiquette of contributing to an open source project. -If you have any questions regarding contributing to this repository please contact the contributor. +All contributors are requested to join this [WhatsApp Group](https://chat.whatsapp.com/L1NrQcgCFWw95FRbytqIJ3) for all forms of communication regarding contributing to this repository. - ### List of major issues > >[!IMPORTANT] @@ -44,7 +43,7 @@ If you have any questions regarding contributing to this repository please conta 1. Make the window resizable and all window elements scalable. 2. Add window control buttons for minimise and fullscreen (to be added after the previous implementation) 3. Whenever you hover over the screenshot button, placeholder text is inserted in the chat box but it doesn't go away after you stop hovering, and you have to manually press backspace to remove the text. -4. Refactor the codebase of the overlay modular so that it is easier to work with. +4. After adding your API Key from the overlay, once the overlay is closed and reopened later, it stops showing the placeholder text that your key has been added. Make the placeholder text persistent across sessions. #### Frontend+Backend :- @@ -52,9 +51,9 @@ If you have any questions regarding contributing to this repository please conta >[!TIP] > Here is a suggested solution: -Store the chats of the user in a database, (we are already using sqlite for screenshots, might as well use it), then when we give a new prompt to gemini, old chats are added to the prompt. +Store the recent chats of the user in a database, (we are already using ChromaDB, might as well use it), then when we give a new prompt to gemini, old chats are added to the prompt. -- Only store the last 30 chats or so. +- Only store the last 30 (or user specified) chats or so. - Every game will have its own database table, i.e. chats are stored on a per game basis. - Additional meta-data such as timestamp should also be stored so when user asked "What did I do yesterday?" it should be able to retrieve the screenshot from 24 hours ago etc. @@ -67,45 +66,52 @@ Store the chats of the user in a database, (we are already using sqlite for scre #### Backend :- -1. Chroma db vector collections aren't searched properly, this may have to do with the chroma client not being initialised properly or the collections are not being created properly in `get_or_create_collection()` or the incorrect implementation of `search_knowledge()` in *vector_service.py*. This issue requires a more thorough investigation. -2. Web Scrapper in *knowledge_manager.py* sometimes gets blocked by certain websites, (*namely* the ones present in *minecraft.csv*) +1. Web Scrappers in *knowledge_manager.py*, `extract_wiki_content()` and `extract_forum_content()` +sometimes gets blocked by certain websites, (*namely* the ones present in *minecraft.csv*) >[!TIP] > Recommended Solutions : - +- Mask the scrappers to behave to more like human by introducing delays between different searches. - Use Proxies to circumvent IP bans. - Rotate a list of User Agents and headers. - Make it asynchronous using `asyncio + httpx` ## List of Improvements and Suggestions -> + >[!TIP] > Feel free to give us any of your ideas, suggestions and feedback to add to this list. +1. **Youtube Video Indexing :** +Add the ability to index the description, audio transcriptions, titles and tags of youtube videos and recommend them to the user for a given prompt, from the list of videos in the CSV file. + +>[!IMPORTANT] +> UI/UX Addition : +> - Add the ability to play the video within the overlay itself. +> - Add basic features play, pause and other functionalities. + ### Improvements to the UI -1. Add Markdown support for Gemini's responses. +1. Add **Markdown support** for Gemini's responses. 2. Improve the chat Window UI such that it is easier to tell apart the messages of the user and Pixly. -3. Add the ability to view images results from the web and play recommended youtube videos directly within the overlay. +3. Add the ability to view **images results from the web**. +4. #### Long-term Goals -1. Add the ability for users to add custom themes of the overlay, different themes belonging to different games, so when the overlay can automatically apply a certain theme when a particular game is detected. +1. Add the ability for users to add **Custom Themes** of the overlay, different themes belonging to different games, so when the overlay can automatically apply a certain theme when a particular game is detected. ### Improvements to Backend :- 1. Add more .csv entries about wikis, guides, youtube videos, forum posts about more games, especially single player story based titles like `Elden Ring, Hollow Knight : Silksong, Black Myth: Wukong,Cyberpunk 2077` -2. Implement a Better way to store screenshots: +2. Implement a better way to store screenshots: >[!TIP] > Suggested Improvements : -- Vectorise the screenshots as well. +- Vectorise the screenshots as well, and store them in ChromaDB (make sure the content is encrypted as well) - Add tool calling for the agent to call a tool to retrieve the screenshot from a specific time or from a specific game. -1. Improve the Web Scrapper : - -In Addition to fixing the above issues, add the ability to scrape youtube audio transcriptions. +2. Add the ability for gemini to automatically perform web searches in case the query that the user is searching isn't present in the data base. If gemini finds said data then it adds to the vector db for future use. ### Long term Goals @@ -125,4 +131,4 @@ In the future we may wanna add cross platform compatibility with Linux. 1. Overlay hangs and then crashes when turning off the *enable screenshots setting.* 2. `game_detection.py`, it reports the incorrect game being detected, in some cases. -3. After adding your API Key from the overlay, sometimes it shows that the user has added their key, sometimes it doesn't. + diff --git a/games_info/minecraft.csv b/games_info/minecraft.csv index 3dc8238..661e56a 100644 --- a/games_info/minecraft.csv +++ b/games_info/minecraft.csv @@ -1,7 +1,5 @@ wiki,wiki_desc,youtube,yt_desc,forum,forum_desc -https://minecraft.wiki/w/Redstone_circuits,Complete guide to redstone mechanics and circuits,https://www.youtube.com/watch?v=dQw4w9WgXcQ,Redstone tutorial for beginners,https://www.reddit.com/r/Minecraft/comments/redstone,Discussion about redstone contraptions +https://minecraft.fandom.com/wiki/Redstone_circuits,Guide to Redstone circuits and mechanisms,https://www.youtube.com/watch?v=R1xI7gI8XzQ,Beginnerโ€™s guide to redstone components and circuits,https://www.reddit.com/r/Minecraft/comments/qugwpp/what_are_the_best_minecraft_redstone_tutorial/,Community picks for the best redstone tutorials +https://minecraft.fandom.com/wiki/Enchanting,Details about enchanting tables and item enhancement,https://www.youtube.com/watch?v=2PCEh7vBB_4,Complete enchanting guide for all enchantments,https://www.reddit.com/r/Minecraft/comments/x9ybaj/enchanting_table_best_setups_and_tips/,Players share best enchanting setups +tps://minecraft.wiki/w/Redstone_circuits,Complete guide to redstone mechanics and circuits,https://www.youtube.com/watch?v=dQw4w9WgXcQ,Redstone tutorial for beginners,https://www.reddit.com/r/Minecraft/comments/redstone,Discussion about redstone contraptions https://minecraft.wiki/w/Enchanting,Everything about enchanting items and books,https://www.youtube.com/watch?v=dQw4w9WgXcQ,Enchanting guide and tips,https://www.reddit.com/r/Minecraft/comments/enchanting,Enchanting strategies and best practices -https://minecraft.wiki/w/Mob,Complete list of all mobs and their behaviors,https://www.youtube.com/watch?v=dQw4w9WgXcQ,Mob farming guide,https://www.reddit.com/r/Minecraft/comments/mobs,Mob spawning and farming discussions -https://minecraft.wiki/w/The_Nether,Nether dimension guide and survival tips,https://www.youtube.com/watch?v=dQw4w9WgXcQ,Nether survival guide,https://www.reddit.com/r/Minecraft/comments/nether,Nether exploration and building tips -https://minecraft.wiki/w/Villager,All about villagers and trading,https://www.youtube.com/watch?v=dQw4w9WgXcQ,Villager trading optimization,https://www.reddit.com/r/Minecraft/comments/villagers,Villager breeding and trading setups - diff --git a/services/chatbot.py b/services/chatbot.py index eec1f70..7562d05 100644 --- a/services/chatbot.py +++ b/services/chatbot.py @@ -23,7 +23,7 @@ def set_api_key(new_key: str): os.environ['GOOGLE_API_KEY'] = new_key genai.configure(api_key=new_key) global model - model = genai.GenerativeModel(model_name="gemini-2.5-flash-lite", system_instruction=system_prompt) + model = genai.GenerativeModel(model_name="gemini-2.5-flash", system_instruction=system_prompt) return True except Exception as e: print(f"Error setting API key: {e}") @@ -88,11 +88,11 @@ async def chat_with_gemini(message: str, image_data: str = None): # Add game context and knowledge if detected if detected_game: - enhanced_message += f"\n\nDETECTED GAME: {detected_game.upper()}" + enhanced_message += f"\n\nDETECTED GAME: {detected_game.lower()}" # Search for relevant knowledge try: - knowledge_results = search_knowledge(detected_game, message, limit=3) + knowledge_results = search_knowledge(detected_game, message) if knowledge_results: knowledge_context = "\n\nRELEVANT KNOWLEDGE FROM GAME DATABASE:\n" diff --git a/services/vector_service.py b/services/vector_service.py index 289a8d5..72a860d 100644 --- a/services/vector_service.py +++ b/services/vector_service.py @@ -52,7 +52,7 @@ def get_or_create_collection(self, game_name: str, content_type: str) -> Optiona if not self.chroma_client: return None - collection_name = f"{game_name}_{content_type}" + collection_name = f"{game_name.lower()}_{content_type.lower()}" try: # Try to get existing collection @@ -63,7 +63,7 @@ def get_or_create_collection(self, game_name: str, content_type: str) -> Optiona # Collection doesn't exist, create it try: collection = self.chroma_client.create_collection( - name=collection_name, + name=collection_name.lower(), metadata={"game": game_name, "content_type": content_type} ) self.collections[collection_name] = collection @@ -197,7 +197,7 @@ def search_knowledge(self, game_name: str, query: str, content_types: List[str] # Search each content type for content_type in content_types: - collection_name = f"{game_name}_{content_type}" + collection_name = f"{game_name.lower()}_{content_type.lower()}" if collection_name in self.collections: collection = self.collections[collection_name] diff --git a/test_system.py b/test_system.py index 0c5d930..932c922 100644 --- a/test_system.py +++ b/test_system.py @@ -91,7 +91,7 @@ def main(): search_data = { "query": query, "limit": 3, - "game_namme":"Minecraft" + "game_name":"Minecraft" } result = test_api_endpoint("/games/minecraft/knowledge/search", "POST", search_data) if result.get("status_code") == 200: diff --git a/test_system_BMW.py b/test_system_BMW.py index 7060ec3..26f63cb 100644 --- a/test_system_BMW.py +++ b/test_system_BMW.py @@ -91,7 +91,6 @@ def main(): search_data = { "game_name": "Black Myth Wukong", "query": query, - "limit": 3 } result = test_api_endpoint("/games/Black_Myth_Wukong/knowledge/search", "POST", search_data) if result.get("status_code") == 200: