{ "issues": [ { "url": "https://api.github.com/repos/Mega4alik/ollm/issues/22", "id": 3485673580, "node_id": "I_kwDOPfKc-c7Pwyhs", "number": 22, "title": "Is there any plan for Quantized model (GGUF or others )?", "user": { "login": "yuhuihu", "id": 4496921, "node_id": "MDQ6VXNlcjQ0OTY5MjE=", "avatar_url": "https://avatars.githubusercontent.com/u/4496921?v=4", "gravatar_id": "", "url": "https://api.github.com/users/yuhuihu", "type": "User", "user_view_type": "public", "site_admin": false }, "labels": {}, "state": "open", "locked": false, "assignee": null, "assignees": {}, "milestone": null, "comments": 0, "created_at": "2025-10-06T04:48:58Z", "updated_at": "2025-10-06T04:48:58Z", "closed_at": null, "author_association": "NONE", "active_lock_reason": null, "sub_issues_summary": { "total": 0, "completed": 0, "percent_completed": 0 }, "issue_dependencies_summary": { "blocked_by": 0, "total_blocked_by": 0, "blocking": 0, "total_blocking": 0 }, "body": "Is there any plan for Quantized model (GGUF or others )?\n\nAs designed for lower hardware from the beginning, supporting for Quantized format will make the project more suitable for production environment. ", "closed_by": null, "reactions": { "url": "https://api.github.com/repos/Mega4alik/ollm/issues/22/reactions", "total_count": 0, "+1": 0, "-1": 0, "laugh": 0, "hooray": 0, "confused": 0, "heart": 0, "rocket": 0, "eyes": 0 }, "performed_via_github_app": null, "state_reason": null, "linked_prs": [] }, { "url": "https://api.github.com/repos/Mega4alik/ollm/issues/21", "id": 3479967127, "node_id": "I_kwDOPfKc-c7PbBWX", "number": 21, "title": "Does it support Arm machine ?", "user": { "login": "mmaaxxii2018", "id": 36187405, "node_id": "MDQ6VXNlcjM2MTg3NDA1", "avatar_url": "https://avatars.githubusercontent.com/u/36187405?v=4", "gravatar_id": "", "url": "https://api.github.com/users/mmaaxxii2018", "type": "User", "user_view_type": "public", "site_admin": false }, "labels": {}, "state": "open", "locked": false, "assignee": null, "assignees": {}, "milestone": null, "comments": 1, "created_at": "2025-10-03T06:32:06Z", "updated_at": "2025-10-03T15:21:10Z", "closed_at": null, "author_association": "NONE", "active_lock_reason": null, "sub_issues_summary": { "total": 0, "completed": 0, "percent_completed": 0 }, "issue_dependencies_summary": { "blocked_by": 0, "total_blocked_by": 0, "blocking": 0, "total_blocking": 0 }, "body": "Thanks for the great work.\nMay i know if it runs on arm machine such as Jetson orin nano ?\nregards,", "closed_by": null, "reactions": { "url": "https://api.github.com/repos/Mega4alik/ollm/issues/21/reactions", "total_count": 0, "+1": 0, "-1": 0, "laugh": 0, "hooray": 0, "confused": 0, "heart": 0, "rocket": 0, "eyes": 0 }, "performed_via_github_app": null, "state_reason": null, "linked_prs": [] }, { "url": "https://api.github.com/repos/Mega4alik/ollm/issues/18", "id": 3469882934, "node_id": "I_kwDOPfKc-c7O0jY2", "number": 18, "title": "Can i contribute to this repo?", "user": { "login": "googbob", "id": 96131018, "node_id": "U_kgDOBbrXyg", "avatar_url": "https://avatars.githubusercontent.com/u/96131018?v=4", "gravatar_id": "", "url": "https://api.github.com/users/googbob", "type": "User", "user_view_type": "public", "site_admin": false }, "labels": {}, "state": "open", "locked": false, "assignee": null, "assignees": {}, "milestone": null, "comments": 2, "created_at": "2025-09-30T14:51:30Z", "updated_at": "2025-10-03T07:40:53Z", "closed_at": null, "author_association": "NONE", "active_lock_reason": null, "sub_issues_summary": { "total": 0, "completed": 0, "percent_completed": 0 }, "issue_dependencies_summary": { "blocked_by": 0, "total_blocked_by": 0, "blocking": 0, "total_blocking": 0 }, "body": null, "closed_by": null, "reactions": { "url": "https://api.github.com/repos/Mega4alik/ollm/issues/18/reactions", "total_count": 1, "+1": 1, "-1": 0, "laugh": 0, "hooray": 0, "confused": 0, "heart": 0, "rocket": 0, "eyes": 0 }, "performed_via_github_app": null, "state_reason": null, "linked_prs": [] }, { "url": "https://api.github.com/repos/Mega4alik/ollm/issues/17", "id": 3469628850, "node_id": "I_kwDOPfKc-c7OzlWy", "number": 17, "title": "Provide OpenAI-alternative API", "user": { "login": "phken91", "id": 8167012, "node_id": "MDQ6VXNlcjgxNjcwMTI=", "avatar_url": "https://avatars.githubusercontent.com/u/8167012?v=4", "gravatar_id": "", "url": "https://api.github.com/users/phken91", "type": "User", "user_view_type": "public", "site_admin": false }, "labels": {}, "state": "open", "locked": false, "assignee": null, "assignees": {}, "milestone": null, "comments": 1, "created_at": "2025-09-30T13:57:51Z", "updated_at": "2025-10-01T02:46:20Z", "closed_at": null, "author_association": "NONE", "active_lock_reason": null, "sub_issues_summary": { "total": 0, "completed": 0, "percent_completed": 0 }, "issue_dependencies_summary": { "blocked_by": 0, "total_blocked_by": 0, "blocking": 0, "total_blocking": 0 }, "body": "I would like to use this library in other services, is there any plan to provide an OpenAI-alternative standard API for external services to use?", "closed_by": null, "reactions": { "url": "https://api.github.com/repos/Mega4alik/ollm/issues/17/reactions", "total_count": 0, "+1": 0, "-1": 0, "laugh": 0, "hooray": 0, "confused": 0, "heart": 0, "rocket": 0, "eyes": 0 }, "performed_via_github_app": null, "state_reason": null, "linked_prs": [] }, { "url": "https://api.github.com/repos/Mega4alik/ollm/issues/16", "id": 3464471856, "node_id": "I_kwDOPfKc-c7Of6Uw", "number": 16, "title": "[design question] why get rid of mmap?", "user": { "login": "gcr", "id": 45252, "node_id": "MDQ6VXNlcjQ1MjUy", "avatar_url": "https://avatars.githubusercontent.com/u/45252?v=4", "gravatar_id": "", "url": "https://api.github.com/users/gcr", "type": "User", "user_view_type": "public", "site_admin": false }, "labels": {}, "state": "open", "locked": false, "assignee": null, "assignees": {}, "milestone": null, "comments": 2, "created_at": "2025-09-29T11:18:12Z", "updated_at": "2025-09-30T18:44:59Z", "closed_at": null, "author_association": "NONE", "active_lock_reason": null, "sub_issues_summary": { "total": 0, "completed": 0, "percent_completed": 0 }, "issue_dependencies_summary": { "blocked_by": 0, "total_blocked_by": 0, "blocking": 0, "total_blocking": 0 }, "body": "Thanks for the library! Really great work to see large models running on consumer devices. :D\n\nI'm curious, why get rid of mmap? Is the kernel page cache really that much less efficient than rolling your own?\n\nMemory allocated with mmap can be evicted from the page cache since it's backed by disk, but memory used with open()/read() must be kept around, so I don't understand why the README says you get lower RAM usage without it. With techniques like madvise(), I expect mmap should have been better on low-end and high-end devices.\n\nWhen llama-cpp switched to mmap-based weight loading in 2023, they found it to be much faster and simpler, even on low-memory devices. See Justine Tunney’s tech report for motivation , https://justine.lol/mmap/ and the llama-cop comments + commit by @jart, https://github.com/ggml-org/llama.cpp/issues/91", "closed_by": null, "reactions": { "url": "https://api.github.com/repos/Mega4alik/ollm/issues/16/reactions", "total_count": 1, "+1": 1, "-1": 0, "laugh": 0, "hooray": 0, "confused": 0, "heart": 0, "rocket": 0, "eyes": 0 }, "performed_via_github_app": null, "state_reason": null, "linked_prs": [] }, { "url": "https://api.github.com/repos/Mega4alik/ollm/issues/14", "id": 3462388996, "node_id": "I_kwDOPfKc-c7OX90E", "number": 14, "title": "Custom resource", "user": { "login": "reploid-cc", "id": 208572657, "node_id": "U_kgDODG6Q8Q", "avatar_url": "https://avatars.githubusercontent.com/u/208572657?v=4", "gravatar_id": "", "url": "https://api.github.com/users/reploid-cc", "type": "User", "user_view_type": "public", "site_admin": false }, "labels": {}, "state": "open", "locked": false, "assignee": null, "assignees": {}, "milestone": null, "comments": 1, "created_at": "2025-09-28T18:59:50Z", "updated_at": "2025-09-28T20:05:32Z", "closed_at": null, "author_association": "NONE", "active_lock_reason": null, "sub_issues_summary": { "total": 0, "completed": 0, "percent_completed": 0 }, "issue_dependencies_summary": { "blocked_by": 0, "total_blocked_by": 0, "blocking": 0, "total_blocking": 0 }, "body": "Is it possible to configure custom resource limits? My system has 64GB RAM + 8GB GPU, but I'd like to cap usage at 40GB RAM and 7.5GB GPU. When running multiple models (2-3 instances), they should prioritize GPU allocation, then use RAM, with SSD as the final resort.", "closed_by": null, "reactions": { "url": "https://api.github.com/repos/Mega4alik/ollm/issues/14/reactions", "total_count": 1, "+1": 1, "-1": 0, "laugh": 0, "hooray": 0, "confused": 0, "heart": 0, "rocket": 0, "eyes": 0 }, "performed_via_github_app": null, "state_reason": null, "linked_prs": [] }, { "url": "https://api.github.com/repos/Mega4alik/ollm/issues/13", "id": 3462251962, "node_id": "I_kwDOPfKc-c7OXcW6", "number": 13, "title": "Voxtral compatibility", "user": { "login": "PhunkyBob", "id": 13345603, "node_id": "MDQ6VXNlcjEzMzQ1NjAz", "avatar_url": "https://avatars.githubusercontent.com/u/13345603?v=4", "gravatar_id": "", "url": "https://api.github.com/users/PhunkyBob", "type": "User", "user_view_type": "public", "site_admin": false }, "labels": {}, "state": "open", "locked": false, "assignee": null, "assignees": {}, "milestone": null, "comments": 3, "created_at": "2025-09-28T17:17:22Z", "updated_at": "2025-09-29T00:22:21Z", "closed_at": null, "author_association": "NONE", "active_lock_reason": null, "sub_issues_summary": { "total": 0, "completed": 0, "percent_completed": 0 }, "issue_dependencies_summary": { "blocked_by": 0, "total_blocked_by": 0, "blocking": 0, "total_blocking": 0 }, "body": "Is there a way to make it run Mistral's Voxtral?", "closed_by": null, "reactions": { "url": "https://api.github.com/repos/Mega4alik/ollm/issues/13/reactions", "total_count": 1, "+1": 1, "-1": 0, "laugh": 0, "hooray": 0, "confused": 0, "heart": 0, "rocket": 0, "eyes": 0 }, "performed_via_github_app": null, "state_reason": null, "linked_prs": [] }, { "url": "https://api.github.com/repos/Mega4alik/ollm/issues/10", "id": 3454389992, "node_id": "I_kwDOPfKc-c7N5c7o", "number": 10, "title": "Mac support", "user": { "login": "marcinkubica", "id": 1593987, "node_id": "MDQ6VXNlcjE1OTM5ODc=", "avatar_url": "https://avatars.githubusercontent.com/u/1593987?v=4", "gravatar_id": "", "url": "https://api.github.com/users/marcinkubica", "type": "User", "user_view_type": "public", "site_admin": false }, "labels": {}, "state": "open", "locked": false, "assignee": null, "assignees": {}, "milestone": null, "comments": 2, "created_at": "2025-09-25T16:44:58Z", "updated_at": "2025-09-28T01:05:13Z", "closed_at": null, "author_association": "NONE", "active_lock_reason": null, "sub_issues_summary": { "total": 0, "completed": 0, "percent_completed": 0 }, "issue_dependencies_summary": { "blocked_by": 0, "total_blocked_by": 0, "blocking": 0, "total_blocking": 0 }, "body": "Hello!\nAny plans for mac? M4 is so nice these days.", "closed_by": null, "reactions": { "url": "https://api.github.com/repos/Mega4alik/ollm/issues/10/reactions", "total_count": 6, "+1": 6, "-1": 0, "laugh": 0, "hooray": 0, "confused": 0, "heart": 0, "rocket": 0, "eyes": 0 }, "performed_via_github_app": null, "state_reason": null, "linked_prs": [] }, { "url": "https://api.github.com/repos/Mega4alik/ollm/issues/8", "id": 3447144776, "node_id": "I_kwDOPfKc-c7Nd0FI", "number": 8, "title": "why ssd? Is RAM possible?", "user": { "login": "ghost", "id": 10137, "node_id": "MDQ6VXNlcjEwMTM3", "avatar_url": "https://avatars.githubusercontent.com/u/10137?v=4", "gravatar_id": "", "url": "https://api.github.com/users/ghost", "type": "User", "user_view_type": "public", "site_admin": false }, "labels": {}, "state": "open", "locked": false, "assignee": null, "assignees": {}, "milestone": null, "comments": 7, "created_at": "2025-09-24T01:02:51Z", "updated_at": "2025-10-03T17:22:08Z", "closed_at": null, "author_association": "NONE", "active_lock_reason": null, "sub_issues_summary": { "total": 0, "completed": 0, "percent_completed": 0 }, "issue_dependencies_summary": { "blocked_by": 0, "total_blocked_by": 0, "blocking": 0, "total_blocking": 0 }, "body": "ssd slow, ram fast. 196g-256g ram is available to consumers.", "closed_by": null, "reactions": { "url": "https://api.github.com/repos/Mega4alik/ollm/issues/8/reactions", "total_count": 1, "+1": 1, "-1": 0, "laugh": 0, "hooray": 0, "confused": 0, "heart": 0, "rocket": 0, "eyes": 0 }, "performed_via_github_app": null, "state_reason": null, "linked_prs": [] } ], "pulls": [], "discussions": [ { "id": "D_kwDOPfKc-c4AhujI", "number": 4, "title": "Which model should we add next?", "body": "Any thoughts are appreciated ", "created_at": "2025-09-04T17:53:59Z", "updated_at": "2025-10-01T08:57:20Z", "category": { "name": "Q&A", "emoji": ":pray:" }, "answer": null, "user": { "login": "Mega4alik", "avatar_url": "https://avatars.githubusercontent.com/u/3070052?u=282bf178a02b73dafa31894dacd15753fda3b27f&v=4" } }, { "id": "D_kwDOPfKc-c4AhuAK", "number": 3, "title": "Llama repetitive outputs", "body": "If you are seeing repetitive outputs from Llama models, update oLLM to the newest 0.2.1 version. Its attention imlementation changed to FlashAttention-2", "created_at": "2025-09-04T07:54:14Z", "updated_at": "2025-09-23T01:52:56Z", "category": { "name": "Announcements", "emoji": ":mega:" }, "answer": null, "user": { "login": "Mega4alik", "avatar_url": "https://avatars.githubusercontent.com/u/3070052?u=282bf178a02b73dafa31894dacd15753fda3b27f&v=4" } }, { "id": "D_kwDOPfKc-c4Ahtv8", "number": 2, "title": "Welcome to ollm Discussions!", "body": "\r\n## 👋 Welcome!\r\n We’re using Discussions as a place to connect with other members of our community. We hope that you:\r\n * Ask questions you’re wondering about.\r\n * Share ideas.\r\n * Engage with other community members.\r\n * Welcome others and are open-minded. Remember that this is a community we\r\n build together 💪.\r\n\r\n To get started, comment below with an introduction of yourself and tell us about what you do with this community.\r\n\r\n\r\n", "created_at": "2025-09-04T01:04:14Z", "updated_at": "2025-09-04T17:54:13Z", "category": { "name": "Announcements", "emoji": ":mega:" }, "answer": null, "user": { "login": "Mega4alik", "avatar_url": "https://avatars.githubusercontent.com/u/3070052?u=282bf178a02b73dafa31894dacd15753fda3b27f&v=4" } } ], "details": { "id": 1039310073, "node_id": "R_kgDOPfKc-Q", "name": "ollm", "full_name": "Mega4alik/ollm", "private": false, "owner": { "login": "Mega4alik", "id": 3070052, "node_id": "MDQ6VXNlcjMwNzAwNTI=", "avatar_url": "https://avatars.githubusercontent.com/u/3070052?v=4", "gravatar_id": "", "url": "https://api.github.com/users/Mega4alik", "type": "User", "user_view_type": "public", "site_admin": false }, "description": null, "fork": false, "url": "https://api.github.com/repos/Mega4alik/ollm", "created_at": "2025-08-16T23:50:10Z", "updated_at": "2025-10-07T03:00:22Z", "pushed_at": "2025-10-05T05:55:53Z", "homepage": null, "size": 240, "stargazers_count": 1633, "watchers_count": 1633, "language": "Python", "has_issues": true, "has_projects": true, "has_downloads": true, "has_wiki": true, "has_pages": false, "has_discussions": true, "forks_count": 134, "archived": false, "disabled": false, "open_issues_count": 9, "license": { "key": "mit", "name": "MIT License", "spdx_id": "MIT", "url": "https://api.github.com/licenses/mit", "node_id": "MDc6TGljZW5zZTEz" }, "allow_forking": true, "is_template": false, "web_commit_signoff_required": false, "topics": {}, "visibility": "public", "forks": 134, "open_issues": 9, "watchers": 1633, "default_branch": "main", "permissions": { "admin": false, "maintain": false, "push": false, "triage": false, "pull": true }, "temp_clone_token": "", "network_count": 134, "subscribers_count": 20 }, "lastFetched": 1759806338233 }