Building AI Agent from Scratch: Minimal Implementation Without Frameworks

Agent frameworks are powerful, but building from scratch is the fastest way to understand the core principles. This guide shows you how in about 130 lines of code.

Core Components

The simplest Agent needs just 4 parts:

1. LLM Engine - model API calls
2. Tool System - dynamic tool registration
3. Memory - context management
4. Loop - control flow

Implementation

1. LLM Engine: Unified Interface

from openai import AsyncOpenAI
from typing import List, Dict, Any

class LLMEngine:
    def __init__(self, model: str = "gpt-4o", api_key: str = None):
        self.client = AsyncOpenAI(api_key=api_key)
        self.model = model

    async def chat(
        self,
        messages: List[Dict],
        tools: List[Dict] = None,
        temperature: float = 0.7
    ) -> Dict[str, Any]:
        response = await self.client.chat.completions.create(
            model=self.model,
            messages=messages,
            tools=tools,
            temperature=temperature
        )

        return {
            "content": response.choices[0].message.content,
            "tool_calls": response.choices[0].message.tool_calls,
            "usage": response.usage.model_dump()
        }

2. Tool System: Dynamic Registration

from typing import Callable, Dict
import inspect

class ToolRegistry:
    def __init__(self):
        self.tools: Dict[str, Callable] = {}

    def register(self, name: str = None):
        def decorator(func: Callable):
            tool_name = name or func.__name__
            self.tools[tool_name] = func
            return func
        return decorator

    def get_tools_schema(self) -> List[Dict]:
        tools = []
        for name, func in self.tools.items():
            sig = inspect.signature(func)
            params = {
                "type": "object",
                "properties": {},
                "required": []
            }
            for pname in sig.parameters:
                params["properties"][pname] = {"type": "string"}
                params["required"].append(pname)

            tools.append({
                "type": "function",
                "function": {
                    "name": name,
                    "description": func.__doc__ or "",
                    "parameters": params
                }
            })
        return tools

    async def call(self, name: str, arguments: Dict) -> Any:
        if name not in self.tools:
            raise ValueError(f"Tool {name} not found")
        return await self.tools[name](**arguments)

3. Memory: Context Management

from collections import deque
from dataclasses import dataclass, field
from typing import List

@dataclass
class Message:
    role: str
    content: str
    tool_calls: List = field(default_factory=list)

class Memory:
    def __init__(self, max_turns: int = 20):
        self.messages: deque = deque(maxlen=max_turns)

    def add(self, role: str, content: str, tool_calls: List = None):
        self.messages.append(Message(role, content, tool_calls or []))

    def to_openai_format(self) -> List[Dict]:
        return [
            {"role": m.role, "content": m.content}
            for m in self.messages
        ]

4. Loop Control

class Agent:
    def __init__(self, llm: LLMEngine, tools: ToolRegistry, max_iterations: int = 10):
        self.llm = llm
        self.tools = tools
        self.max_iterations = max_iterations
        self.memory = Memory()

    async def run(self, prompt: str) -> str:
        self.memory.add("user", prompt)

        for i in range(self.max_iterations):
            tools_schema = self.tools.get_tools_schema()

            response = await self.llm.chat(
                self.memory.to_openai_format(),
                tools=tools_schema
            )

            if response["tool_calls"]:
                tool_call = response["tool_calls"][0]
                tool_name = tool_call.function.name
                arguments = eval(f"dict({tool_call.function.arguments})")

                self.memory.add("assistant", f"Using tool: {tool_name}", [tool_call])

                result = await self.tools.call(tool_name, arguments)

                self.memory.add("tool", str(result))
            else:
                self.memory.add("assistant", response["content"])
                return response["content"]

        return "Max iterations reached"

Usage Example

# Create Agent
llm = LLMEngine(model="gpt-4o")
tools = ToolRegistry()

@tools.register("calculate")
def calculate(expression: str) -> str:
    """Execute math calculation"""
    return str(eval(expression))

@tools.register("weather")
def weather(city: str) -> str:
    """Query weather"""
    return f"{city} is sunny, 25 degrees"

agent = Agent(llm, tools)

# Run
result = asyncio.run(agent.run("What's the weather like in Beijing?"))
print(result)

Summary

Total approximately 130 lines. You can use frameworks like LangChain later, but understanding the principles lets you customize better.

Learning path: Run this minimal version first, then add streaming, reflection mechanisms, and multimodal capabilities step by step.