const axios = require('axios');

// Configuration
const API_KEY = 'YOUR_API_KEY';
const BASE_URL = 'https://app.dumplingai.com/api/v1';
const KNOWLEDGE_BASE_ID = 'YOUR_KNOWLEDGE_BASE_ID'; // Create this in the dashboard

// Helper function for API requests
async function callDumplingAPI(endpoint, data) {
  try {
    const response = await axios.post(`${BASE_URL}/${endpoint}`, data, {
      headers: {
        'Content-Type': 'application/json',
        'Authorization': `Bearer ${API_KEY}`
      }
    });
    return response.data;
  } catch (error) {
    console.error(`Error calling ${endpoint}:`, error.response ? error.response.data : error.message);
    throw error;
  }
}

async function scrapeWebsite(url) {
  console.log(`Scraping website: ${url}`);
  
  const result = await callDumplingAPI('scrape', {
    url: url,
    format: 'markdown',
    cleaned: true,
    renderJs: true
  });
  
  console.log(`Successfully scraped: ${result.title}`);
  return result;
}

// Usage
const url = 'https://example.com/article';
const scrapedContent = await scrapeWebsite(url);

async function extractKeyInformation(content, title) {
  console.log(`Extracting key information from: ${title}`);
  
  const prompt = `
    Extract the most important information from the following content. 
    Format your response as JSON with the following structure:
    {
      "title": "The main title",
      "summary": "A concise summary (max 150 words)",
      "keyPoints": ["point 1", "point 2", "point 3"],
      "entities": [{"name": "Entity name", "type": "person/organization/location/etc"}],
      "categories": ["category1", "category2"]
    }
    
    Content:
    ${content}
  `;
  
  const result = await callDumplingAPI('agents/generate-completion', {
    messages: [
      {
        role: 'user',
        content: prompt
      }
    ],
    agentId: 'YOUR_AGENT_ID', // Replace with your agent ID
    parseJson: true
  });
  
  console.log(`Successfully extracted information with ${result.parsedJson.keyPoints.length} key points`);
  return result.parsedJson;
}

// Usage
const extractedInfo = await extractKeyInformation(scrapedContent.content, scrapedContent.title);

async function storeInKnowledgeBase(extractedInfo, sourceUrl) {
  console.log(`Storing information in knowledge base...`);
  
  // Format content for the knowledge base
  const formattedContent = `
    # ${extractedInfo.title}
    
    ## Summary
    ${extractedInfo.summary}
    
    ## Key Points
    ${extractedInfo.keyPoints.map(point => `- ${point}`).join('\n')}
    
    ## Categories
    ${extractedInfo.categories.join(', ')}
  `;
  
  const result = await callDumplingAPI('knowledge-bases/add', {
    knowledgeBaseId: KNOWLEDGE_BASE_ID,
    content: formattedContent,
    metadata: {
      title: extractedInfo.title,
      source: sourceUrl,
      extractedAt: new Date().toISOString(),
      categories: extractedInfo.categories,
      entities: extractedInfo.entities
    }
  });
  
  console.log(`Successfully stored in knowledge base with ID: ${result.id}`);
  return result;
}

// Usage
const storedResult = await storeInKnowledgeBase(extractedInfo, url);

async function searchKnowledgeBase(query) {
  console.log(`Searching knowledge base for: ${query}`);
  
  const result = await callDumplingAPI('knowledge-bases/search', {
    knowledgeBaseId: KNOWLEDGE_BASE_ID,
    query: query,
    maxResults: 5
  });
  
  console.log(`Found ${result.results.length} results for query: ${query}`);
  
  // Format and display results
  result.results.forEach((item, index) => {
    console.log(`\nResult ${index + 1} (Score: ${item.score.toFixed(2)}):`);
    console.log(`Title: ${item.metadata.title}`);
    console.log(`Source: ${item.metadata.source}`);
    console.log(`Extracted: ${new Date(item.metadata.extractedAt).toLocaleString()}`);
    console.log(`Categories: ${item.metadata.categories.join(', ')}`);
    console.log(`\nContent Preview: ${item.content.substring(0, 200)}...`);
  });
  
  return result.results;
}

// Usage
const searchResults = await searchKnowledgeBase("analytics dashboard features");

async function contentExtractionWorkflow(urls) {
  const processedUrls = [];
  
  for (const url of urls) {
    try {
      console.log(`\n==== Processing URL: ${url} ====\n`);
      
      // Step 1: Scrape website
      const scrapedContent = await scrapeWebsite(url);
      
      // Step 2: Extract key information
      const extractedInfo = await extractKeyInformation(
        scrapedContent.content, 
        scrapedContent.title
      );
      
      // Step 3: Store in knowledge base
      const storedResult = await storeInKnowledgeBase(extractedInfo, url);
      
      processedUrls.push({
        url,
        title: extractedInfo.title,
        success: true,
        knowledgeBaseItemId: storedResult.id
      });
      
    } catch (error) {
      console.error(`Error processing URL ${url}:`, error);
      processedUrls.push({
        url,
        success: false,
        error: error.message
      });
    }
  }
  
  console.log(`\n==== Processing Complete ====\n`);
  console.log(`Successfully processed ${processedUrls.filter(u => u.success).length} out of ${urls.length} URLs`);
  
  return processedUrls;
}

// Example usage
const urlsToProcess = [
  'https://example.com/article1',
  'https://example.com/article2',
  'https://example.com/blog/post1'
];

const results = await contentExtractionWorkflow(urlsToProcess);

// Search the processed content
const searchResults = await searchKnowledgeBase("machine learning trends");