SDK

// node.js
import { pipe } from '@screenpipe/js'

// browser
import { pipe } from '@screenpipe/browser'

const results = await pipe.queryScreenpipe({
  q: "john",
  contentType: "ocr", // "ocr" | "audio" | "ui" | "all" | "audio+ui" | "ocr+ui" | "audio+ocr"
  limit: 10,
  offset: 0,
  startTime: "2024-03-10T12:00:00Z",
  endTime: "2024-03-10T13:00:00Z",
  appName: "chrome",
  windowName: "meeting",
  includeFrames: true,
  minLength: 10,
  maxLength: 1000,
  speakerIds: [1, 2],
  frameName: "screenshot.png",
  browserUrl: "github.com/mediar-ai/screenpipe" // Filter by browser URL
})

// Get the last hour of screen activity from Chrome
const oneHourAgo = new Date(Date.now() - 60 * 60 * 1000).toISOString()
const now = new Date().toISOString()

const results = await pipe.queryScreenpipe({
  contentType: "ocr",
  startTime: oneHourAgo,
  endTime: now,
  appName: "chrome",
  limit: 50,
  includeFrames: true, // include base64 encoded images
})

// Process results
for (const item of results.data) {
  console.log(`At ${item.content.timestamp}:`)
  console.log(`Text: ${item.content.text}`)
  
  if (item.content.frame) {
    // You could display or process the base64 image
    // e.g., <img src={`data:image/png;base64,${item.content.frame}`} />
  }
}

const results = await pipe.queryScreenpipe({
  browserUrl: "discord*",
  startTime: new Date(Date.now() - 1000 * 60 * 60 * 24).toISOString(),
  limit: 20,
})

// Process results based on content type, when using browserUrl, only OCR is available
for (const item of results.data) {
  if (item.type === "OCR") {
    console.log(`Screen text: ${item.content.text}`)
  } else if (item.type === "Audio") {
    console.log(`Audio transcript: ${item.content.transcription}`)
    console.log(`Speaker: ${item.content.speaker_id}`)
  } else if (item.type === "UI") {
    console.log(`UI element: ${item.content.element_type}`)
  }
}

// Get a full day of activity
const startOfDay = new Date()
startOfDay.setHours(0, 0, 0, 0)

const endOfDay = new Date() 
endOfDay.setHours(23, 59, 59, 999)

// Get both screen and audio data
const results = await pipe.queryScreenpipe({
  contentType: "ocr+audio",
  startTime: startOfDay.toISOString(),
  endTime: endOfDay.toISOString(),
  limit: 1000, // Get a large sample
})

// Group by hour for timeline visualization
const timelineByHour = {}

for (const item of results.data) {
  const timestamp = new Date(item.content.timestamp)
  const hour = timestamp.getHours()
  
  // Initialize hour bucket if needed
  if (!timelineByHour[hour]) {
    timelineByHour[hour] = {
      screenEvents: 0,
      audioEvents: 0,
      apps: new Set(),
    }
  }
  
  // Update counts
  if (item.type === "OCR") {
    timelineByHour[hour].screenEvents++
    
    if (item.content.app_name) {
      timelineByHour[hour].apps.add(item.content.app_name)
    }
  } else if (item.type === "Audio") {
    timelineByHour[hour].audioEvents++
  }
}

// Now timelineByHour can be used to build a visualization
console.log(timelineByHour)

/**
 * Retrieves all content captured from a specific website
 * This is useful for aggregating research materials or tracking 
 * activity on particular web applications
 */
async function getWebsiteCaptures(domain: string, days = 7) {
  const startTime = new Date(Date.now() - days * 24 * 60 * 60 * 1000).toISOString();
  const endTime = new Date().toISOString();

  const results = await pipe.queryScreenpipe({
    browserUrl: domain,  // Target specific website
    contentType: "ocr",  // Get both visual content and UI elements
    startTime,
    endTime,
    limit: 100,
    includeFrames: true  // Include screenshots
  });

  // Process and analyze the results
  const captures = results.data.map(item => {
    const timestamp = new Date(
      item.type === "OCR" ? item.content.timestamp : item.content.timestamp
    );
    
    return {
      timestamp,
      date: timestamp.toLocaleDateString(),
      time: timestamp.toLocaleTimeString(),
      type: item.type,
      content: item.type === "OCR" ? item.content.text : item.content.text,
      screenshot: item.type === "OCR" && item.content.frame ? item.content.frame : null,
      url: item.type === "OCR" ? item.content.browserUrl : item.content.browserUrl
    };
  });

  return captures;
}

// Example: Get all GitHub activity from the past week
const githubActivity = await getWebsiteCaptures("github.com");
console.log(`Found ${githubActivity.length} interactions with GitHub`);

// Example: Search for specific text within a website
const searchTerm = "pull request";
const pullRequests = githubActivity.filter(item => 
  item.content.toLowerCase().includes(searchTerm.toLowerCase())
);
console.log(`Found ${pullRequests.length} pull request interactions`);

{
  "crons": [
    {
      "path": "/api/log",
      "schedule": "0 */5 * * *" // every 5 minutes
    }
  ]
}

bunx @screenpipe/dev@latest components add
# select "update-pipe-config" from the menu

// stream transcriptions
for await (const chunk of pipe.streamTranscriptions()) {
  console.log(chunk.choices[0].text)
  console.log(chunk.metadata) // { timestamp, device, isInput }
}

// stream vision events
for await (const event of pipe.streamVision(true)) { // true to include images
  console.log(event.data.text)
  console.log(event.data.app_name)
  console.log(event.data.image) // base64 if includeImages=true
}

bunx --bun @screenpipe/dev@latest components add

import OpenAI from 'openai'

const openai = new OpenAI({
  apiKey: process.env.OPENAI_API_KEY,
})

async function summarizeMeeting() {
  let transcript = ""
  let summaryBuffer = ""
  let lastSummaryTime = Date.now()
  
  // Stream transcriptions in real-time
  for await (const chunk of pipe.streamTranscriptions()) {
    const text = chunk.choices[0].text
    transcript += text + " "
    
    // Generate summary every 30 seconds
    if (Date.now() - lastSummaryTime > 30000 && transcript.length > 200) {
      const summary = await openai.chat.completions.create({
        model: "gpt-3.5-turbo",
        messages: [
          {
            role: "system",
            content: "You summarize meeting transcripts concisely."
          },
          {
            role: "user",
            content: `Summarize this meeting transcript: ${transcript}`
          }
        ]
      })
      
      summaryBuffer = summary.choices[0].message.content
      lastSummaryTime = Date.now()
      
      // Display or store the summary
      console.log("Meeting Summary Update:", summaryBuffer)
    }
  }
}

// Start summarizing
summarizeMeeting().catch(console.error)

async function monitorScreenActivity() {
  // Monitor screen content in real-time
  for await (const event of pipe.streamVision(true)) {
    // Check if we're looking at important apps
    const appName = event.data.app_name?.toLowerCase() || ""
    const text = event.data.text?.toLowerCase() || ""
    
    // Example: Detect when looking at email
    if (
      appName.includes("outlook") || 
      appName.includes("gmail") ||
      text.includes("inbox") ||
      text.includes("compose email")
    ) {
      console.log("Email activity detected at", new Date().toLocaleTimeString())
      
      // You could log this, send notifications, etc.
    }
    
    // Example: Track time spent in different applications
    // This would require keeping state between events
  }
}

// Start monitoring
monitorScreenActivity().catch(console.error)

bunx --bun @screenpipe/dev@latest components add
# select "use-pipe-settings" from the menu

// src/lib/types.ts
import type { Settings as ScreenpipeAppSettings } from '@screenpipe/js'

export interface Settings {
  // your pipe specific settings
  customSetting?: string
  anotherSetting?: number
  
  // screenpipe app settings
  screenpipeAppSettings?: ScreenpipeAppSettings
}

// src/lib/actions/get-screenpipe-app-settings.ts
import { pipe } from '@screenpipe/js'
import type { Settings as ScreenpipeAppSettings } from '@screenpipe/js'

export async function getScreenpipeAppSettings() {
  return await pipe.settings.getAll()
}

export async function updateScreenpipeAppSettings(
  newSettings: Partial<ScreenpipeAppSettings>
) {
  return await pipe.settings.update(newSettings)
}

// src/lib/hooks/use-pipe-settings.tsx
import { useState, useEffect } from 'react'
import { Settings } from '@/lib/types'
import {
  getScreenpipeAppSettings,
  updateScreenpipeAppSettings,
} from '@/lib/actions/get-screenpipe-app-settings'

const DEFAULT_SETTINGS: Partial<Settings> = {
  customSetting: 'default value',
  anotherSetting: 42,
}

export function usePipeSettings() {
  const [settings, setSettings] = useState<Partial<Settings> | null>(null)
  const [loading, setLoading] = useState(true)

  useEffect(() => {
    loadSettings()
  }, [])

  const loadSettings = async () => {
    try {
      // load screenpipe app settings
      const screenpipeSettings = await getScreenpipeAppSettings()

      // get pipe specific settings from customSettings
      const pipeSettings = {
        ...(screenpipeSettings.customSettings?.yourPipeName && {
          ...screenpipeSettings.customSettings.yourPipeName,
        }),
      }

      // merge everything together
      setSettings({
        ...DEFAULT_SETTINGS,
        ...pipeSettings,
        screenpipeAppSettings: screenpipeSettings,
      })
    } catch (error) {
      console.error('failed to load settings:', error)
    } finally {
      setLoading(false)
    }
  }

  const updateSettings = async (newSettings: Partial<Settings>) => {
    try {
      // split settings
      const { screenpipeAppSettings, ...pipeSettings } = newSettings

      const mergedPipeSettings = {
        ...DEFAULT_SETTINGS,
        ...pipeSettings,
      }

      // update screenpipe settings if provided
      await updateScreenpipeAppSettings({
        ...screenpipeAppSettings,
        customSettings: {
          ...screenpipeAppSettings?.customSettings,
          yourPipeName: pipeSettings,
        },
      })

      // update state with everything
      setSettings({
        ...mergedPipeSettings,
        screenpipeAppSettings:
          screenpipeAppSettings || settings?.screenpipeAppSettings,
      })
      return true
    } catch (error) {
      console.error('failed to update settings:', error)
      return false
    }
  }

  return { settings, updateSettings, loading }
}

import { usePipeSettings } from '@/lib/hooks/use-pipe-settings'

export function SettingsComponent() {
  const { settings, updateSettings, loading } = usePipeSettings()

  if (loading) return <div>loading...</div>

  return (
    <form onSubmit={async (e) => {
      e.preventDefault()
      const formData = new FormData(e.target as HTMLFormElement)
      await updateSettings({
        customSetting: formData.get('customSetting') as string,
        anotherSetting: parseInt(formData.get('anotherSetting') as string),
      })
    }}>
      <input 
        name="customSetting"
        defaultValue={settings?.customSetting}
      />
      <input 
        name="anotherSetting"
        type="number"
        defaultValue={settings?.anotherSetting}
      />
      <button type="submit">save</button>
    </form>
  )
}

import { generateObject, embed } from "ai";
import { ollama } from "ollama-ai-provider";
import { pipe } from "@screenpipe/js";
import { z } from "zod";

// define your output schema
const workLog = z.object({
  title: z.string(),
  description: z.string(),
  tags: z.array(z.string()),
  mediaLinks: z.array(z.string()).optional(),
});

type WorkLog = z.infer<typeof workLog> & {
  startTime: string;
  endTime: string;
};

async function generateWorkLog(screenData, model, startTime, endTime) {
  // configure the prompt with context and instructions
  const prompt = `You are analyzing screen recording data from Screenpipe.
    Based on the following screen data, generate a concise work activity log entry.
    
    Screen data: ${JSON.stringify(screenData)}
    
    Return a JSON object with:
    {
        "title": "Brief title describing the main activity",
        "description": "Clear description of what was accomplished",
        "tags": ["#relevant-tool", "#activity-type"],
        "mediaLinks": ["<video src=\"file:///path/to/video.mp4\" controls></video>"]
    }`;

  // use ollama provider with vercel ai sdk
  const provider = ollama(model);
  const response = await generateObject({
    model: provider,
    messages: [{ role: "user", content: prompt }],
    schema: workLog, // zod schema for type safety
  });

  return {
    ...response.object,
    startTime: formatDate(startTime),
    endTime: formatDate(endTime),
  };
}

// api endpoint implementation
async function handleRequest() {
  // get last hour of activity
  const now = new Date();
  const oneHourAgo = new Date(now.getTime() - 3600000);

  // query context from screenpipe
  const screenData = await pipe.queryScreenpipe({
    startTime: oneHourAgo.toISOString(),
    endTime: now.toISOString(),
    limit: 100,
    contentType: "all",
  });

  // generate structured log with ai
  const logEntry = await generateWorkLog(
    screenData.data,
    "llama3.2", // ollama model name
    oneHourAgo,
    now
  );

  return { message: "log generated successfully", logEntry };
}

import { embed } from "ai";
import { ollama } from "ollama-ai-provider";
import { ContentItem } from "@screenpipe/js";

async function deduplicateScreenData(screenData: ContentItem[]): Promise<ContentItem[]> {
  if (!screenData.length) return screenData;

  // use ollama's embedding model
  const provider = ollama.embedding("nomic-embed-text");
  const embeddings: number[][] = [];
  const uniqueData: ContentItem[] = [];
  
  for (const item of screenData) {
    // extract text content depending on type
    const textToEmbed =
      "content" in item
        ? typeof item.content === "string"
          ? item.content
          : "text" in item.content
          ? item.content.text
          : JSON.stringify(item.content)
        : "";

    if (!textToEmbed.trim()) {
      uniqueData.push(item);
      continue;
    }

    // generate embedding
    const { embedding } = await embed({
      model: provider,
      value: textToEmbed,
    });

    // check for duplicates using cosine similarity
    let isDuplicate = false;
    for (let i = 0; i < embeddings.length; i++) {
      const similarity = cosineSimilarity(embedding, embeddings[i]);
      if (similarity > 0.95) {
        isDuplicate = true;
        break;
      }
    }

    if (!isDuplicate) {
      embeddings.push(embedding);
      uniqueData.push(item);
    }
  }

  return uniqueData;
}

// cosine similarity helper
function cosineSimilarity(a: number[], b: number[]): number {
  const dotProduct = a.reduce((sum, val, i) => sum + val * b[i], 0);
  const normA = Math.sqrt(a.reduce((sum, val) => sum + val * val, 0));
  const normB = Math.sqrt(b.reduce((sum, val) => sum + val * val, 0));
  return dotProduct / (normA * normB);
}

import { pipe } from "@screenpipe/js";
import { OpenAI } from "openai";
import type { Settings } from "@screenpipe/js";


async function streamMeetingInsights(settings: Settings) {
  const openai = new OpenAI({
    apiKey:
      settings?.aiProviderType === "screenpipe-cloud"
        ? settings?.user?.token
        : settings?.openaiApiKey,
    baseURL: settings?.aiUrl,
    dangerouslyAllowBrowser: true, // for browser environments
  });
  let transcript = "";
  
  // stream audio transcriptions
  for await (const chunk of pipe.streamTranscriptions()) {
    transcript += chunk.choices[0].text + " ";
    
    // analyze every 30 seconds
    if (transcript.length > 200 && transcript.length % 30 === 0) {
      const response = await openai.chat.completions.create({
        model: "gpt-4o", // if user is using screenpipe-cloud, you can also use claude-3-7-sonnet or gemini-2.0-flash-lite
        messages: [
          {
            role: "system",
            content: "provide brief insights on this ongoing meeting."
          },
          {
            role: "user", 
            content: transcript
          }
        ]
      });
      
      console.log("meeting insight:", response.choices[0].message.content);
    }
  }
}

await pipe.sendDesktopNotification({
  title: "meeting starting",
  body: "your standup begins in 5 minutes",
})

// settings management
const settings = await pipe.settings.getAll()
await pipe.settings.update({ aiModel: "gpt-4" })

// inbox management (node only)
const messages = await pipe.inbox.getMessages()
await pipe.inbox.clearMessages()