SHA256
Initial vibe-coded commit.
This commit is contained in:
Discsearcher
@@ -0,0 +1,299 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
markmywords - Self-hosted markdown repository viewer with search
|
||||
"""
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import json
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from flask import Flask, render_template, request, jsonify
|
||||
from apscheduler.schedulers.background import BackgroundScheduler
|
||||
from apscheduler.triggers.cron import CronTrigger
|
||||
import markdown
|
||||
import logging
|
||||
from whoosh.index import create_in, open_dir
|
||||
from whoosh.fields import Schema, TEXT, ID
|
||||
from whoosh.qparser import QueryParser
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
# Configuration
|
||||
CONFIG_FILE = "/config/repositories.json"
|
||||
REPOS_DIR = "/data/repos"
|
||||
INDEX_DIR = "/data/search_index"
|
||||
MARKDOWN_EXTENSIONS = ['.md', '.markdown']
|
||||
|
||||
# Setup logging
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Ensure directories exist
|
||||
os.makedirs(REPOS_DIR, exist_ok=True)
|
||||
os.makedirs(INDEX_DIR, exist_ok=True)
|
||||
|
||||
# Initialize search index
|
||||
def create_search_index():
|
||||
"""Create or open the search index."""
|
||||
schema = Schema(
|
||||
path=ID(stored=True),
|
||||
title=TEXT(stored=True),
|
||||
content=TEXT(stored=False)
|
||||
)
|
||||
|
||||
if not os.path.exists(os.path.join(INDEX_DIR, "WRITELOCK")):
|
||||
return create_in(INDEX_DIR, schema)
|
||||
return open_dir(INDEX_DIR)
|
||||
|
||||
ix = create_search_index()
|
||||
|
||||
def load_repositories():
|
||||
"""Load repository configuration from file."""
|
||||
if not os.path.exists(CONFIG_FILE):
|
||||
logger.warning(f"Config file not found: {CONFIG_FILE}")
|
||||
return {}
|
||||
|
||||
try:
|
||||
with open(CONFIG_FILE, 'r') as f:
|
||||
return json.load(f)
|
||||
except Exception as e:
|
||||
logger.error(f"Error loading config: {e}")
|
||||
return {}
|
||||
|
||||
def git_pull_repo(repo_name, repo_path):
|
||||
"""Perform a git pull on the specified repository."""
|
||||
try:
|
||||
logger.info(f"Pulling repository: {repo_name}")
|
||||
result = subprocess.run(
|
||||
["git", "pull"],
|
||||
cwd=repo_path,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=300
|
||||
)
|
||||
logger.info(f"Pull result for {repo_name}: {result.stdout}")
|
||||
if result.returncode != 0:
|
||||
logger.error(f"Pull error for {repo_name}: {result.stderr}")
|
||||
return result.returncode == 0
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.error(f"Git pull timeout for {repo_name}")
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"Error pulling {repo_name}: {e}")
|
||||
return False
|
||||
|
||||
def clone_or_pull(repo_name, repo_url):
|
||||
"""Clone repository if it doesn't exist, otherwise pull."""
|
||||
repo_path = os.path.join(REPOS_DIR, repo_name)
|
||||
|
||||
if not os.path.exists(repo_path):
|
||||
try:
|
||||
logger.info(f"Cloning repository: {repo_name} from {repo_url}")
|
||||
subprocess.run(
|
||||
["git", "clone", repo_url, repo_path],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=300
|
||||
)
|
||||
logger.info(f"Successfully cloned {repo_name}")
|
||||
except Exception as e:
|
||||
logger.error(f"Error cloning {repo_name}: {e}")
|
||||
return False
|
||||
else:
|
||||
return git_pull_repo(repo_name, repo_path)
|
||||
|
||||
return True
|
||||
|
||||
def update_search_index():
|
||||
"""Update the search index with all markdown files."""
|
||||
try:
|
||||
writer = ix.writer()
|
||||
|
||||
for repo_dir in Path(REPOS_DIR).iterdir():
|
||||
if not repo_dir.is_dir():
|
||||
continue
|
||||
|
||||
for md_file in repo_dir.rglob('*'):
|
||||
if md_file.suffix.lower() in MARKDOWN_EXTENSIONS:
|
||||
try:
|
||||
content = md_file.read_text(encoding='utf-8', errors='ignore')
|
||||
# Extract title from filename or first heading
|
||||
title = md_file.stem
|
||||
rel_path = str(md_file.relative_to(REPOS_DIR))
|
||||
|
||||
writer.add_document(
|
||||
path=rel_path,
|
||||
title=title,
|
||||
content=content
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Error indexing {md_file}: {e}")
|
||||
|
||||
writer.commit()
|
||||
logger.info("Search index updated successfully")
|
||||
except Exception as e:
|
||||
logger.error(f"Error updating search index: {e}")
|
||||
|
||||
def scheduled_pull():
|
||||
"""Perform scheduled pulls of all repositories."""
|
||||
logger.info("Starting scheduled repository pull")
|
||||
repos = load_repositories()
|
||||
|
||||
for repo_name, repo_config in repos.items():
|
||||
if not repo_config.get('enabled', True):
|
||||
continue
|
||||
|
||||
clone_or_pull(repo_name, repo_config['url'])
|
||||
|
||||
# Update search index after pulling
|
||||
update_search_index()
|
||||
logger.info("Scheduled pull completed")
|
||||
|
||||
def setup_scheduler():
|
||||
"""Setup the background scheduler for periodic pulls."""
|
||||
scheduler = BackgroundScheduler()
|
||||
repos = load_repositories()
|
||||
|
||||
for repo_name, repo_config in repos.items():
|
||||
if repo_config.get('enabled', True):
|
||||
cron_schedule = repo_config.get('schedule', '0 */6 * * *') # Default: every 6 hours
|
||||
try:
|
||||
scheduler.add_job(
|
||||
scheduled_pull,
|
||||
CronTrigger.from_crontab(cron_schedule),
|
||||
id=f"pull_{repo_name}",
|
||||
name=f"Pull {repo_name}"
|
||||
)
|
||||
logger.info(f"Scheduled pull for {repo_name}: {cron_schedule}")
|
||||
except Exception as e:
|
||||
logger.error(f"Error scheduling {repo_name}: {e}")
|
||||
|
||||
scheduler.start()
|
||||
return scheduler
|
||||
|
||||
# Routes
|
||||
@app.route('/')
|
||||
def index():
|
||||
"""Display the main page with list of repositories and search."""
|
||||
repos = load_repositories()
|
||||
repo_list = []
|
||||
|
||||
for repo_name in repos.keys():
|
||||
repo_path = os.path.join(REPOS_DIR, repo_name)
|
||||
markdown_files = []
|
||||
|
||||
if os.path.exists(repo_path):
|
||||
for md_file in Path(repo_path).rglob('*'):
|
||||
if md_file.suffix.lower() in MARKDOWN_EXTENSIONS:
|
||||
rel_path = str(md_file.relative_to(repo_path))
|
||||
markdown_files.append({
|
||||
'name': md_file.stem,
|
||||
'path': rel_path
|
||||
})
|
||||
|
||||
repo_list.append({
|
||||
'name': repo_name,
|
||||
'files': sorted(markdown_files, key=lambda x: x['path'])
|
||||
})
|
||||
|
||||
return render_template('index.html', repositories=repo_list)
|
||||
|
||||
@app.route('/api/search', methods=['GET'])
|
||||
def search():
|
||||
"""Search markdown files."""
|
||||
query_str = request.args.get('q', '').strip()
|
||||
|
||||
if not query_str or len(query_str) < 2:
|
||||
return jsonify({'results': []})
|
||||
|
||||
try:
|
||||
with ix.searcher() as searcher:
|
||||
query = QueryParser("content", ix.schema).parse(query_str)
|
||||
results = searcher.search(query)
|
||||
|
||||
search_results = []
|
||||
for hit in results:
|
||||
search_results.append({
|
||||
'path': hit['path'],
|
||||
'title': hit['title']
|
||||
})
|
||||
|
||||
return jsonify({'results': search_results})
|
||||
except Exception as e:
|
||||
logger.error(f"Search error: {e}")
|
||||
return jsonify({'results': [], 'error': str(e)})
|
||||
|
||||
@app.route('/view/<repo>/<path:filepath>')
|
||||
def view_file(repo, filepath):
|
||||
"""View a markdown file converted to HTML."""
|
||||
# Security: prevent directory traversal
|
||||
if '..' in filepath or filepath.startswith('/'):
|
||||
return "Invalid path", 400
|
||||
|
||||
file_path = os.path.join(REPOS_DIR, repo, filepath)
|
||||
|
||||
# Ensure the file is within the repo directory
|
||||
try:
|
||||
file_path = os.path.realpath(file_path)
|
||||
repo_path = os.path.realpath(os.path.join(REPOS_DIR, repo))
|
||||
if not file_path.startswith(repo_path):
|
||||
return "Access denied", 403
|
||||
except Exception:
|
||||
return "Invalid path", 400
|
||||
|
||||
if not os.path.exists(file_path):
|
||||
return "File not found", 404
|
||||
|
||||
try:
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
md_content = f.read()
|
||||
|
||||
# Convert markdown to HTML
|
||||
html_content = markdown.markdown(
|
||||
md_content,
|
||||
extensions=['extra', 'codehilite', 'toc']
|
||||
)
|
||||
|
||||
return render_template(
|
||||
'view.html',
|
||||
repo=repo,
|
||||
filepath=filepath,
|
||||
content=html_content,
|
||||
filename=os.path.basename(filepath)
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Error reading file {file_path}: {e}")
|
||||
return f"Error reading file: {e}", 500
|
||||
|
||||
@app.route('/api/status')
|
||||
def status():
|
||||
"""Return application status."""
|
||||
repos = load_repositories()
|
||||
repo_status = {}
|
||||
|
||||
for repo_name in repos.keys():
|
||||
repo_path = os.path.join(REPOS_DIR, repo_name)
|
||||
repo_status[repo_name] = {
|
||||
'cloned': os.path.exists(repo_path),
|
||||
'last_modified': datetime.fromtimestamp(
|
||||
os.path.getmtime(repo_path)
|
||||
).isoformat() if os.path.exists(repo_path) else None
|
||||
}
|
||||
|
||||
return jsonify({
|
||||
'status': 'running',
|
||||
'repositories': repo_status,
|
||||
'timestamp': datetime.now().isoformat()
|
||||
})
|
||||
|
||||
if __name__ == '__main__':
|
||||
# Perform initial pull and index
|
||||
logger.info("Initializing markmywords")
|
||||
scheduled_pull()
|
||||
|
||||
# Setup scheduler
|
||||
scheduler = setup_scheduler()
|
||||
|
||||
# Start Flask app
|
||||
app.run(host='0.0.0.0', port=5000, debug=False)
|
||||
Reference in New Issue
Block a user