|
|
@@ -1,4 +1,5 @@
|
|
|
import { serve } from "https://deno.land/std@0.168.0/http/server.ts";
|
|
|
+import { isValidExternalUrl, verifyAuth } from '../_shared/security.ts';
|
|
|
|
|
|
const corsHeaders = {
|
|
|
'Access-Control-Allow-Origin': '*',
|
|
|
@@ -18,29 +19,64 @@ serve(async (req) => {
|
|
|
}
|
|
|
|
|
|
try {
|
|
|
+ // Authentication: Require authenticated user
|
|
|
+ const auth = await verifyAuth(req);
|
|
|
+ if (!auth) {
|
|
|
+ console.log('Unauthorized access attempt to fetch-website-rss');
|
|
|
+ return new Response(
|
|
|
+ JSON.stringify({ success: false, error: 'Unauthorized' }),
|
|
|
+ { status: 401, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
|
|
|
+ );
|
|
|
+ }
|
|
|
+
|
|
|
const { url } = await req.json();
|
|
|
|
|
|
- if (!url) {
|
|
|
+ if (!url || typeof url !== 'string') {
|
|
|
+ return new Response(
|
|
|
+ JSON.stringify({ success: false, error: 'Valid URL is required' }),
|
|
|
+ { status: 400, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
|
|
|
+ );
|
|
|
+ }
|
|
|
+
|
|
|
+ // SSRF Protection: Validate URL
|
|
|
+ const urlValidation = isValidExternalUrl(url);
|
|
|
+ if (!urlValidation.valid) {
|
|
|
+ console.log(`SSRF blocked in fetch-website-rss: ${url} - ${urlValidation.error}`);
|
|
|
return new Response(
|
|
|
- JSON.stringify({ error: 'URL is required' }),
|
|
|
+ JSON.stringify({ success: false, error: urlValidation.error }),
|
|
|
{ status: 400, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
|
|
|
);
|
|
|
}
|
|
|
|
|
|
console.log('Fetching website RSS for URL:', url);
|
|
|
|
|
|
- // Fetch the webpage
|
|
|
- const response = await fetch(url, {
|
|
|
- headers: {
|
|
|
- 'User-Agent': 'Mozilla/5.0 (compatible; RSS Feed Detector/1.0)',
|
|
|
- },
|
|
|
- });
|
|
|
+ // Fetch the webpage with timeout
|
|
|
+ const controller = new AbortController();
|
|
|
+ const timeoutId = setTimeout(() => controller.abort(), 30000); // 30s timeout
|
|
|
+
|
|
|
+ let response;
|
|
|
+ try {
|
|
|
+ response = await fetch(url, {
|
|
|
+ signal: controller.signal,
|
|
|
+ headers: {
|
|
|
+ 'User-Agent': 'Mozilla/5.0 (compatible; RSS Feed Detector/1.0)',
|
|
|
+ },
|
|
|
+ });
|
|
|
+ } finally {
|
|
|
+ clearTimeout(timeoutId);
|
|
|
+ }
|
|
|
|
|
|
if (!response.ok) {
|
|
|
throw new Error(`Failed to fetch website: ${response.status}`);
|
|
|
}
|
|
|
|
|
|
const html = await response.text();
|
|
|
+
|
|
|
+ // Limit content size
|
|
|
+ if (html.length > 5 * 1024 * 1024) { // 5MB limit
|
|
|
+ throw new Error('Website content too large');
|
|
|
+ }
|
|
|
+
|
|
|
const feeds: RSSFeed[] = [];
|
|
|
|
|
|
// Extract site name from <title> tag
|
|
|
@@ -54,12 +90,12 @@ serve(async (req) => {
|
|
|
for (const link of links) {
|
|
|
const typeMatch = link.match(/type=["']([^"']+)["']/i);
|
|
|
const hrefMatch = link.match(/href=["']([^"']+)["']/i);
|
|
|
- const titleMatch = link.match(/title=["']([^"']+)["']/i);
|
|
|
+ const linkTitleMatch = link.match(/title=["']([^"']+)["']/i);
|
|
|
|
|
|
if (typeMatch && hrefMatch) {
|
|
|
const type = typeMatch[1];
|
|
|
const href = hrefMatch[1];
|
|
|
- const title = titleMatch ? titleMatch[1] : 'RSS Feed';
|
|
|
+ const title = linkTitleMatch ? linkTitleMatch[1] : 'RSS Feed';
|
|
|
|
|
|
if (
|
|
|
type.includes('application/rss+xml') ||
|
|
|
@@ -69,11 +105,15 @@ serve(async (req) => {
|
|
|
// Convert relative URLs to absolute
|
|
|
const feedUrl = href.startsWith('http') ? href : new URL(href, url).toString();
|
|
|
|
|
|
- feeds.push({
|
|
|
- url: feedUrl,
|
|
|
- title,
|
|
|
- type,
|
|
|
- });
|
|
|
+ // Validate the discovered feed URL too
|
|
|
+ const feedUrlValidation = isValidExternalUrl(feedUrl);
|
|
|
+ if (feedUrlValidation.valid) {
|
|
|
+ feeds.push({
|
|
|
+ url: feedUrl,
|
|
|
+ title,
|
|
|
+ type,
|
|
|
+ });
|
|
|
+ }
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
@@ -85,8 +125,21 @@ serve(async (req) => {
|
|
|
|
|
|
for (const path of commonPaths) {
|
|
|
const testUrl = `${baseUrl.origin}${path}`;
|
|
|
+
|
|
|
+ // Validate fallback URL
|
|
|
+ const testUrlValidation = isValidExternalUrl(testUrl);
|
|
|
+ if (!testUrlValidation.valid) continue;
|
|
|
+
|
|
|
try {
|
|
|
- const testResponse = await fetch(testUrl, { method: 'HEAD' });
|
|
|
+ const testController = new AbortController();
|
|
|
+ const testTimeoutId = setTimeout(() => testController.abort(), 5000); // 5s timeout for fallback checks
|
|
|
+
|
|
|
+ const testResponse = await fetch(testUrl, {
|
|
|
+ method: 'HEAD',
|
|
|
+ signal: testController.signal
|
|
|
+ });
|
|
|
+ clearTimeout(testTimeoutId);
|
|
|
+
|
|
|
if (testResponse.ok) {
|
|
|
feeds.push({
|
|
|
url: testUrl,
|
|
|
@@ -101,7 +154,7 @@ serve(async (req) => {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
- console.log('Found feeds:', feeds);
|
|
|
+ console.log('Found feeds:', feeds.length);
|
|
|
|
|
|
if (feeds.length === 0) {
|
|
|
return new Response(
|
|
|
@@ -130,7 +183,7 @@ serve(async (req) => {
|
|
|
return new Response(
|
|
|
JSON.stringify({
|
|
|
success: false,
|
|
|
- error: (error as Error).message || 'Failed to detect RSS feed'
|
|
|
+ error: 'Failed to detect RSS feed'
|
|
|
}),
|
|
|
{ status: 500, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
|
|
|
);
|