Data Extraction Agent

Active Agent provides data extraction capabilities to parse structured data from unstructured text, images, or PDFs.

Setup

Generate a data extraction agent:

rails generate active_agent:agent data_extraction parse_content

Agent Implementation

data_extraction_agent.rb

class DataExtractionAgent < ApplicationAgent
  before_action :set_multimodal_content, only: [ :parse_content ]

  def parse_content
    prompt(
      message: params[:message] || "Parse the content of the file or image",
      image_data: @image_data,
      file_data: @file_data,
      output_schema: params[:output_schema]
      )
  end

  def describe_cat_image
    prompt(
      message: "Describe the cat in the image",
      image_data: CatImageService.fetch_base64_image
      )
  end

  private
  def set_multimodal_content
    if params[:file_path].present?
      @file_data ||= "data:application/pdf;base64,#{Base64.encode64(File.read(params[:file_path]))}"
    elsif params[:image_path].present?
      @image_data ||= "data:image/jpeg;base64,#{Base64.encode64(File.read(params[:image_path]))}"
    end
  end
end

chart_schema.json.erb

{
  "format": {
    "type": "json_schema",
    "name": "chart_schema",
    "schema": {
      "type": "object",
      "properties": {
        "title": {
          "type": "string",
          "description": "The title of the chart."
        },
        "data_points": {
          "type": "array",
          "items": {
            "$ref": "#/$defs/data_point"
          }
        }
      },
      "required": ["title", "data_points"],
      "additionalProperties": false,
      "$defs": {
        "data_point": {
          "type": "object",
          "properties": {
            "label": {
              "type": "string",
              "description": "The label for the data point."
            },
            "value": {
              "type": "number",
              "description": "The value of the data point."
            }
          },
          "required": ["label", "value"],
          "additionalProperties": false
        }
      }
    }
  }
}

resume_schema.json.erb

{
  "format": {
    "type": "json_schema",
    "name": "resume_schema",
    "schema": {
      "type": "object",
      "properties": {
        "name": {
          "type": "string",
          "description": "The full name of the individual."
        },
        "email": {
          "type": "string",
          "format": "email",
          "description": "The email address of the individual."
        },
        "phone": {
          "type": "string",
          "description": "The phone number of the individual."
        },
        "education": {
          "type": "array",
          "items": {
            "$ref": "#/$defs/education"
          }
        },
        "experience": {
          "type": "array",
          "items": {
            "$ref": "#/$defs/experience"
          }
        }
      },
      "required": ["name", "email", "phone", "education", "experience"],
      "additionalProperties": false,
      "$defs": {
        "education": {
          "type": "object",
          "properties": {
            "degree": {
              "type": "string",
              "description": "The degree obtained."
            },
            "institution": {
              "type": "string",
              "description": "The institution where the degree was obtained."
            },
            "year": {
              "type": "integer",
              "description": "The year of graduation."
            }
          },
          "required": ["degree", "institution", "year"],
          "additionalProperties": false
        },
        "experience": {
          "type": "object",
          "properties": {
            "job_title": {
              "type": "string",
              "description": "The job title held."
            },
            "company": {
              "type": "string",
              "description": "The company where the individual worked."
            },
            "duration": {
              "type": "string",
              "description": "The duration of employment."
            }
          },
          "required": ["job_title", "company", "duration"],
          "additionalProperties": false
        }
      }
    },
    "strict": true
  }
}

Basic Image Example

Image Description

Active Agent can extract descriptions from images without structured output:

prompt = DataExtractionAgent.describe_cat_image

Basic Cat Image Response Example

activeagent/test/agents/data_extraction_agent_test.rb:21

# Response object
#<ActiveAgent::GenerationProvider::Response:0x40c4
  @message=#<ActiveAgent::ActionPrompt::Message:0x40d8
    @action_id=nil,
    @action_name=nil,
    @action_requested=false,
    @charset="UTF-8",
    @content="The cat in the image is lying on its back on a brown leather surface. It has a primarily white coat with some black patches. Its paws are stretched out, and the cat appears to be comfortably relaxed, with its eyes closed and a peaceful expression. The ligh...",
    @role=:assistant>
  @prompt=#<ActiveAgent::ActionPrompt::Prompt:0x40ec ...>
  @content_type="text/plain"
  @raw_response={...}>

# Message content
response.message.content # => "The cat in the image is lying on its back on a brown leather surface. It has a primarily white coat with some black patches. Its paws are stretched out, and the cat appears to be comfortably relaxed, with its eyes closed and a peaceful expression. The light from the sun creates a warm glow around it, highlighting its features."

Image: Parse Chart Data

Active Agent can extract data from chart images:

prompt = DataExtractionAgent.with(
  image_path: sales_chart_path
).parse_content

Basic Chart Image Response Example

activeagent/test/agents/data_extraction_agent_test.rb:107

# Response object
#<ActiveAgent::GenerationProvider::Response:0x4164
  @message=#<ActiveAgent::ActionPrompt::Message:0x4178
    @action_id=nil,
    @action_name=nil,
    @action_requested=false,
    @charset="UTF-8",
    @content="The image presents a bar chart titled \"Quarterly Sales Report\" for the year 2024. It depicts sales revenue by quarter, with data represented for four quarters (Q1, Q2, Q3, and Q4) using differently colored bars:\n\n- **Q1**: Blue bar\n- **Q2**: Green bar\n- **...",
    @role=:assistant>
  @prompt=#<ActiveAgent::ActionPrompt::Prompt:0x418c ...>
  @content_type="text/plain"
  @raw_response={...}>

# Message content
response.message.content # => "The image presents a bar chart titled \"Quarterly Sales Report\" for the year 2024. It depicts sales revenue by quarter, with data represented for four quarters (Q1, Q2, Q3, and Q4) using differently colored bars:\n\n- **Q1**: Blue bar\n- **Q2**: Green bar\n- **Q3**: Yellow bar\n- **Q4**: Red bar\n\nThe sales revenue ranges from $0 to $100,000, with each quarter showing varying levels of sales revenue, with Q4 having the highest bar."

Structured Output

Active Agent supports structured output using JSON schemas. Define schemas in your agent's views directory (e.g., app/views/data_extraction_agent/) and reference them using the output_schema parameter.

Structured Output Schemas

When using structured output:

• The response will have content_type of application/json
• The response content will be valid JSON matching your schema
• Parse the response with JSON.parse(response.message.content)

Parse Chart Image with Structured Output

Extract chart data with a predefined schema chart_schema:

data_extraction_agent_test.rb

prompt = DataExtractionAgent.with(
  output_schema: :chart_schema,
  image_path: sales_chart_path
).parse_content

chart_schema.json.erb

{
  "format": {
    "type": "json_schema",
    "name": "chart_schema",
    "schema": {
      "type": "object",
      "properties": {
        "title": {
          "type": "string",
          "description": "The title of the chart."
        },
        "data_points": {
          "type": "array",
          "items": {
            "$ref": "#/$defs/data_point"
          }
        }
      },
      "required": ["title", "data_points"],
      "additionalProperties": false,
      "$defs": {
        "data_point": {
          "type": "object",
          "properties": {
            "label": {
              "type": "string",
              "description": "The label for the data point."
            },
            "value": {
              "type": "number",
              "description": "The value of the data point."
            }
          },
          "required": ["label", "value"],
          "additionalProperties": false
        }
      }
    }
  }
}

Response

response = prompt.generate_now

Generation Response Example

activeagent/test/agents/data_extraction_agent_test.rb:139

# Response object
#<ActiveAgent::GenerationProvider::Response:0x3fe8
  @message=#<ActiveAgent::ActionPrompt::Message:0x3ffc
    @action_id=nil,
    @action_name=nil,
    @action_requested=false,
    @charset="UTF-8",
    @content="{\"title\":\"Quarterly Sales Report\",\"data_points\":[{\"label\":\"Q1\",\"value\":25000},{\"label\":\"Q2\",\"value\":50000},{\"label\":\"Q3\",\"value\":75000},{\"label\":\"Q4\",\"value\":100000}]}",
    @role=:assistant>
  @prompt=#<ActiveAgent::ActionPrompt::Prompt:0x4010 ...>
  @content_type="application/json"
  @raw_response={...}>

# Message content
response.message.content # => "{\"title\":\"Quarterly Sales Report\",\"data_points\":[{\"label\":\"Q1\",\"value\":25000},{\"label\":\"Q2\",\"value\":50000},{\"label\":\"Q3\",\"value\":75000},{\"label\":\"Q4\",\"value\":100000}]}"

Parse Resume with output resume schema

Extract information from PDF resumes:

data_extraction_agent_test.rb

prompt = DataExtractionAgent.with(
  output_schema: :resume_schema,
  file_path: sample_resume_path
).parse_content

resume_schema.json.erb

{
  "format": {
    "type": "json_schema",
    "name": "resume_schema",
    "schema": {
      "type": "object",
      "properties": {
        "name": {
          "type": "string",
          "description": "The full name of the individual."
        },
        "email": {
          "type": "string",
          "format": "email",
          "description": "The email address of the individual."
        },
        "phone": {
          "type": "string",
          "description": "The phone number of the individual."
        },
        "education": {
          "type": "array",
          "items": {
            "$ref": "#/$defs/education"
          }
        },
        "experience": {
          "type": "array",
          "items": {
            "$ref": "#/$defs/experience"
          }
        }
      },
      "required": ["name", "email", "phone", "education", "experience"],
      "additionalProperties": false,
      "$defs": {
        "education": {
          "type": "object",
          "properties": {
            "degree": {
              "type": "string",
              "description": "The degree obtained."
            },
            "institution": {
              "type": "string",
              "description": "The institution where the degree was obtained."
            },
            "year": {
              "type": "integer",
              "description": "The year of graduation."
            }
          },
          "required": ["degree", "institution", "year"],
          "additionalProperties": false
        },
        "experience": {
          "type": "object",
          "properties": {
            "job_title": {
              "type": "string",
              "description": "The job title held."
            },
            "company": {
              "type": "string",
              "description": "The company where the individual worked."
            },
            "duration": {
              "type": "string",
              "description": "The duration of employment."
            }
          },
          "required": ["job_title", "company", "duration"],
          "additionalProperties": false
        }
      }
    },
    "strict": true
  }
}

Parse Resume with Structured Output

Extract resume data with a predefined resume_schema:

response = prompt.generate_now

Generation Response Example

activeagent/test/agents/data_extraction_agent_test.rb:76

# Response object
#<ActiveAgent::GenerationProvider::Response:0x404c
  @message=#<ActiveAgent::ActionPrompt::Message:0x4060
    @action_id=nil,
    @action_name=nil,
    @action_requested=false,
    @charset="UTF-8",
    @content="{\"name\":\"John Doe\",\"email\":\"john.doe@example.com\",\"phone\":\"(555) 123-4567\",\"education\":[{\"degree\":\"BS Computer Science\",\"institution\":\"Stanford University\",\"year\":2020}],\"experience\":[{\"job_title\":\"Senior Software Engineer\",\"company\":\"TechCorp\",\"duration\":...",
    @role=:assistant>
  @prompt=#<ActiveAgent::ActionPrompt::Prompt:0x4074 ...>
  @content_type="application/json"
  @raw_response={...}>

# Message content
response.message.content # => "{\"name\":\"John Doe\",\"email\":\"john.doe@example.com\",\"phone\":\"(555) 123-4567\",\"education\":[{\"degree\":\"BS Computer Science\",\"institution\":\"Stanford University\",\"year\":2020}],\"experience\":[{\"job_title\":\"Senior Software Engineer\",\"company\":\"TechCorp\",\"duration\":\"2020-2024\"}]}"