from flask import Flask, jsonify, request
import sqlite3
import requests
import os
import json
import time
from datetime import datetime, timedelta
from threading import Thread, Lock
import logging
import hashlib

app = Flask(__name__)
DB_PATH = os.path.join(os.path.dirname(__file__), 'dramalive.db')
JSON_DIR = os.path.join(os.path.dirname(__file__), 'json_data')

# إضافة Lock لمنع مشكلة database locked
db_lock = Lock()

# إعدادات التسجيل مع دعم UTF-8
logging.basicConfig(
    level=logging.INFO, 
    format='%(asctime)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger(__name__)

def init_directories():
    """إنشاء المجلدات اللازمة"""
    os.makedirs(JSON_DIR, exist_ok=True)

def init_database():
    """تهيئة قاعدة البيانات SQLite"""
    with db_lock:
        conn = sqlite3.connect(DB_PATH)
        cursor = conn.cursor()
        
        # جدول المسلسلات
        cursor.execute('''
            CREATE TABLE IF NOT EXISTS series (
                id INTEGER PRIMARY KEY,
                title TEXT NOT NULL,
                description TEXT,
                year INTEGER,
                imdb REAL,
                rating REAL,
                duration TEXT,
                classification TEXT,
                image TEXT,
                cover TEXT,
                trailer_url TEXT,
                page_number INTEGER,
                data_hash TEXT,
                is_new BOOLEAN DEFAULT 1,
                last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
            )
        ''')
        
        # جدول المواسم
        cursor.execute('''
            CREATE TABLE IF NOT EXISTS seasons (
                id INTEGER PRIMARY KEY,
                series_id INTEGER,
                title TEXT NOT NULL,
                season_order INTEGER,
                data_hash TEXT,
                last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                FOREIGN KEY (series_id) REFERENCES series (id) ON DELETE CASCADE
            )
        ''')
        
        # جدول الحلقات
        cursor.execute('''
            CREATE TABLE IF NOT EXISTS episodes (
                id INTEGER PRIMARY KEY,
                season_id INTEGER,
                title TEXT NOT NULL,
                description TEXT,
                duration TEXT,
                episode_order INTEGER,
                data_hash TEXT,
                last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                FOREIGN KEY (season_id) REFERENCES seasons (id) ON DELETE CASCADE
            )
        ''')
        
        # جدول روابط الحلقات
        cursor.execute('''
            CREATE TABLE IF NOT EXISTS episode_sources (
                id INTEGER PRIMARY KEY,
                episode_id INTEGER,
                title TEXT NOT NULL,
                url TEXT NOT NULL,
                quality TEXT,
                source_order INTEGER,
                data_hash TEXT,
                last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                FOREIGN KEY (episode_id) REFERENCES episodes (id) ON DELETE CASCADE
            )
        ''')
        
        # جدول الأفلام
        cursor.execute('''
            CREATE TABLE IF NOT EXISTS movies (
                id INTEGER PRIMARY KEY,
                title TEXT NOT NULL,
                description TEXT,
                year INTEGER,
                imdb REAL,
                rating REAL,
                duration TEXT,
                classification TEXT,
                image TEXT,
                cover TEXT,
                trailer_url TEXT,
                page_number INTEGER,
                data_hash TEXT,
                is_new BOOLEAN DEFAULT 1,
                last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
            )
        ''')
        
        # جدول روابط الأفلام
        cursor.execute('''
            CREATE TABLE IF NOT EXISTS movie_sources (
                id INTEGER PRIMARY KEY,
                movie_id INTEGER,
                title TEXT NOT NULL,
                url TEXT NOT NULL,
                quality TEXT,
                source_order INTEGER,
                data_hash TEXT,
                last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                FOREIGN KEY (movie_id) REFERENCES movies (id) ON DELETE CASCADE
            )
        ''')
        
        # جدول لتتبع حالة الجلب
        cursor.execute('''
            CREATE TABLE IF NOT EXISTS crawl_progress (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                data_type TEXT NOT NULL,
                total_pages INTEGER,
                completed_pages INTEGER,
                last_page_crawled INTEGER,
                status TEXT DEFAULT 'in_progress',
                start_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                end_time TIMESTAMP,
                items_added INTEGER DEFAULT 0,
                items_updated INTEGER DEFAULT 0
            )
        ''')
        
        # إنشاء الفهرس للترتيب السريع
        cursor.execute('CREATE INDEX IF NOT EXISTS idx_series_page ON series(page_number)')
        cursor.execute('CREATE INDEX IF NOT EXISTS idx_series_updated ON series(last_updated)')
        cursor.execute('CREATE INDEX IF NOT EXISTS idx_movies_page ON movies(page_number)')
        cursor.execute('CREATE INDEX IF NOT EXISTS idx_movies_updated ON movies(last_updated)')
        cursor.execute('CREATE INDEX IF NOT EXISTS idx_series_new ON series(is_new)')
        cursor.execute('CREATE INDEX IF NOT EXISTS idx_movies_new ON movies(is_new)')
        
        conn.commit()
        conn.close()
    logger.info("✅ تم تهيئة قاعدة البيانات SQLite")

def calculate_data_hash(data):
    """حساب هاش للبيانات لاكتشاف التغييرات مع دعم UTF-8"""
    data_str = json.dumps(data, sort_keys=True, ensure_ascii=False)
    return hashlib.md5(data_str.encode('utf-8')).hexdigest()

def save_to_json(data_type, data, page_number):
    """حفظ البيانات في ملف JSON مع دعم UTF-8 وإضافة الـ Sources"""
    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
    filename = f"{data_type}_page_{page_number}_{timestamp}.json"
    filepath = os.path.join(JSON_DIR, filename)
    
    try:
        # نسخة معدلة من البيانات مع إضافة الـ Sources
        enhanced_data = []
        
        for item in data:
            # نسخ العنصر الأساسي
            enhanced_item = item.copy()
            
            if data_type == 'movies':
                # جلب sources الفيلم وإضافتها
                try:
                    sources_url = f'https://app.arabypros.com/api/movie/source/by/{item["id"]}/4F5A9C3D9A86FA54EACEDDD635185/d506abfd-9fe2-4b71-b979-feff21bcad13/'
                    sources_response = requests.get(sources_url, timeout=15)
                    sources_response.raise_for_status()
                    sources = sources_response.json()
                    enhanced_item['sources'] = sources
                    logger.info(f"🎬 تم إضافة {len(sources)} مصدر للفيلم {item['id']} - {item['title']}")
                except requests.RequestException as e:
                    logger.warning(f"⚠️ لم يتم جلب مصادر الفيلم {item['id']}: {e}")
                    enhanced_item['sources'] = []
            
            elif data_type == 'series':
                # جلب المواسم والحلقات والمصادر للمسلسل
                try:
                    seasons_url = f'https://app.arabypros.com/api/season/by/serie/{item["id"]}/4F5A9C3D9A86FA54EACEDDD635185/d506abfd-9fe2-4b71-b979-feff21bcad13/'
                    seasons_response = requests.get(seasons_url, timeout=20)
                    seasons_response.raise_for_status()
                    seasons = seasons_response.json()
                    
                    # إضافة المصادر لكل حلقة
                    seasons_with_sources = []
                    for season in seasons:
                        season_copy = season.copy()
                        episodes_with_sources = []
                        
                        for episode in season.get('episodes', []):
                            episode_copy = episode.copy()
                            try:
                                episode_sources_url = f'https://app.arabypros.com/api/episode/source/by/{episode["id"]}/4F5A9C3D9A86FA54EACEDDD635185/d506abfd-9fe2-4b71-b979-feff21bcad13/'
                                episode_sources_response = requests.get(episode_sources_url, timeout=15)
                                episode_sources_response.raise_for_status()
                                episode_copy['sources'] = episode_sources_response.json()
                            except requests.RequestException as e:
                                logger.warning(f"⚠️ لم يتم جلب مصادر الحلقة {episode['id']}: {e}")
                                episode_copy['sources'] = []
                            
                            episodes_with_sources.append(episode_copy)
                        
                        season_copy['episodes'] = episodes_with_sources
                        seasons_with_sources.append(season_copy)
                    
                    enhanced_item['seasons'] = seasons_with_sources
                    total_episodes = sum(len(season['episodes']) for season in seasons_with_sources)
                    logger.info(f"📺 تم إضافة {len(seasons_with_sources)} موسم و {total_episodes} حلقة للمسلسل {item['id']} - {item['title']}")
                    
                except requests.RequestException as e:
                    logger.warning(f"⚠️ لم يتم جلب مواسم المسلسل {item['id']}: {e}")
                    enhanced_item['seasons'] = []
            
            enhanced_data.append(enhanced_item)
        
        # حفظ البيانات المحسنة
        with open(filepath, 'w', encoding='utf-8') as f:
            json.dump(enhanced_data, f, ensure_ascii=False, indent=2)
        logger.info(f"💾 تم حفظ بيانات {data_type} صفحة {page_number} في {filename} (مع المصادر)")
        return filename
        
    except Exception as e:
        logger.error(f"❌ خطأ في حفظ JSON: {e}")
        return None

def update_crawl_progress(data_type, current_page, total_pages, items_added=0, items_updated=0, status='in_progress'):
    """تحديث تقدم عملية الجلب"""
    try:
        with db_lock:
            conn = sqlite3.connect(DB_PATH)
            cursor = conn.cursor()
            
            if status == 'completed':
                cursor.execute('''
                    INSERT INTO crawl_progress 
                    (data_type, total_pages, completed_pages, last_page_crawled, status, end_time, items_added, items_updated)
                    VALUES (?, ?, ?, ?, ?, CURRENT_TIMESTAMP, ?, ?)
                ''', (data_type, total_pages, total_pages, current_page, status, items_added, items_updated))
            else:
                # البحث عن السجل النشط
                cursor.execute('''
                    SELECT id FROM crawl_progress 
                    WHERE data_type = ? AND status = 'in_progress'
                    ORDER BY start_time DESC LIMIT 1
                ''', (data_type,))
                
                result = cursor.fetchone()
                if result:
                    cursor.execute('''
                        UPDATE crawl_progress 
                        SET completed_pages = ?, last_page_crawled = ?, items_added = items_added + ?, items_updated = items_updated + ?
                        WHERE id = ?
                    ''', (current_page + 1, current_page, items_added, items_updated, result[0]))
                else:
                    cursor.execute('''
                        INSERT INTO crawl_progress 
                        (data_type, total_pages, completed_pages, last_page_crawled, items_added, items_updated)
                        VALUES (?, ?, ?, ?, ?, ?)
                    ''', (data_type, total_pages, current_page + 1, current_page, items_added, items_updated))
            
            conn.commit()
            conn.close()
    except Exception as e:
        logger.error(f"❌ خطأ في تحديث تقدم الجلب: {e}")

def fetch_and_store_series_data(total_pages, is_update=False):
    """جلب وتخزين بيانات المسلسلات"""
    operation_type = "تحديث" if is_update else "جلب"
    logger.info(f"🔁 بدء {operation_type} {total_pages} صفحة من المسلسلات...")
    
    total_new_items = 0
    total_updated_items = 0
    
    # تحديد نطاق الصفحات للجلب
    if is_update:
        # في وضع التحديث: نفحص فقط أحدث الصفحات
        start_page = max(0, 84 - total_pages)  # افتراض أن 84 هو أقصى صفحة
        end_page = 84
        pages_to_fetch = list(range(start_page, end_page + 1))
        logger.info(f"🔄 فحص أحدث {total_pages} صفحة مسلسلات: من {start_page} إلى {end_page}")
    else:
        # في وضع الجلب الكامل: كل الصفحات من الأحدث إلى الأقدم
        pages_to_fetch = list(range(total_pages - 1, -1, -1))
        logger.info(f"📥 جلب كامل لـ {total_pages} صفحة مسلسلات (من الأحدث إلى الأقدم)")
    
    for page_number in pages_to_fetch:
        logger.info(f"📺 جلب صفحة المسلسلات {page_number + 1}")
        
        try:
            url = f'https://app.arabypros.com/api/poster/by/filtres/24/created/{page_number}/4F5A9C3D9A86FA54EACEDDD635185/d506abfd-9fe2-4b71-b979-feff21bcad13/'
            response = requests.get(url, timeout=30)
            response.raise_for_status()
            series_data = response.json()
            
            # حفظ البيانات في JSON مع دعم UTF-8
            save_to_json('series', series_data, page_number)
            
        except requests.RequestException as e:
            logger.error(f"❌ خطأ في جلب صفحة المسلسلات {page_number}: {e}")
            continue

        page_new_items = 0
        page_updated_items = 0
        
        for series in series_data:
            try:
                # استخدام Lock لمنع مشكلة database locked
                with db_lock:
                    conn = sqlite3.connect(DB_PATH)
                    cursor = conn.cursor()
                    
                    # حساب هاش البيانات لاكتشاف التغييرات
                    series_hash = calculate_data_hash(series)
                    
                    # التحقق مما إذا كانت البيانات موجودة ومتطابقة
                    cursor.execute('SELECT data_hash, is_new FROM series WHERE id = ?', (series['id'],))
                    existing_data = cursor.fetchone()
                    
                    if existing_data:
                        existing_hash, is_new = existing_data
                        if existing_hash == series_hash:
                            # البيانات لم تتغير
                            conn.close()
                            continue
                        else:
                            # البيانات تغيرت - تحديث
                            cursor.execute('''
                                UPDATE series SET 
                                title=?, description=?, year=?, imdb=?, rating=?, duration=?, 
                                classification=?, image=?, cover=?, trailer_url=?, page_number=?, 
                                data_hash=?, last_updated=CURRENT_TIMESTAMP, is_new=0
                                WHERE id=?
                            ''', (
                                series['title'], series['description'], series['year'],
                                series['imdb'], series['rating'], series['duration'],
                                series['classification'], series['image'], series.get('cover'),
                                series.get('trailer', {}).get('url'), page_number, series_hash, series['id']
                            ))
                            page_updated_items += 1
                            logger.info(f"🔄 تم تحديث المسلسل: {series['title']}")
                    else:
                        # بيانات جديدة - إدخال
                        cursor.execute('''
                            INSERT INTO series 
                            (id, title, description, year, imdb, rating, duration, classification, 
                             image, cover, trailer_url, page_number, data_hash, is_new)
                            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1)
                        ''', (
                            series['id'], series['title'], series['description'], series['year'],
                            series['imdb'], series['rating'], series['duration'], series['classification'],
                            series['image'], series.get('cover'), series.get('trailer', {}).get('url'),
                            page_number, series_hash
                        ))
                        page_new_items += 1
                        logger.info(f"✅ تم إضافة مسلسل جديد: {series['title']}")
                    
                    # جلب بيانات المواسم
                    try:
                        seasons_url = f'https://app.arabypros.com/api/season/by/serie/{series["id"]}/4F5A9C3D9A86FA54EACEDDD635185/d506abfd-9fe2-4b71-b979-feff21bcad13/'
                        seasons_response = requests.get(seasons_url, timeout=30)
                        seasons_response.raise_for_status()
                        seasons = seasons_response.json()
                        logger.info(f"📺 تم جلب {len(seasons)} موسم للمسلسل {series['id']}")
                    except requests.RequestException as e:
                        logger.error(f"❌ خطأ في جلب مواسم المسلسل {series['id']}: {e}")
                        seasons = []

                    for season_index, season in enumerate(seasons):
                        season_hash = calculate_data_hash(season)
                        cursor.execute('''
                            INSERT OR REPLACE INTO seasons 
                            (id, series_id, title, season_order, data_hash, last_updated)
                            VALUES (?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
                        ''', (
                            season['id'],
                            series['id'],
                            season['title'],
                            season_index,
                            season_hash
                        ))

                        # جلب بيانات الحلقات
                        for episode_index, episode in enumerate(season.get('episodes', [])):
                            episode_hash = calculate_data_hash(episode)
                            cursor.execute('''
                                INSERT OR REPLACE INTO episodes 
                                (id, season_id, title, description, duration, episode_order, data_hash, last_updated)
                                VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
                            ''', (
                                episode['id'],
                                season['id'],
                                episode['title'],
                                episode['description'],
                                episode['duration'],
                                episode_index,
                                episode_hash
                            ))

                            # جلب روابط التشغيل للحلقات
                            try:
                                sources_url = f'https://app.arabypros.com/api/episode/source/by/{episode["id"]}/4F5A9C3D9A86FA54EACEDDD635185/d506abfd-9fe2-4b71-b979-feff21bcad13/'
                                sources_response = requests.get(sources_url, timeout=30)
                                sources_response.raise_for_status()
                                sources = sources_response.json()
                                logger.info(f"🔗 تم جلب {len(sources)} رابط تشغيل للحلقة {episode['id']}")
                            except requests.RequestException as e:
                                logger.error(f"❌ خطأ في جلب روابط الحلقة {episode['id']}: {e}")
                                sources = []

                            for source_index, source in enumerate(sources):
                                source_hash = calculate_data_hash(source)
                                cursor.execute('''
                                    INSERT OR REPLACE INTO episode_sources 
                                    (id, episode_id, title, url, quality, source_order, data_hash, last_updated)
                                    VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
                                ''', (
                                    source['id'],
                                    episode['id'],
                                    source['title'],
                                    source['url'],
                                    source['quality'],
                                    source_index,
                                    source_hash
                                ))

                    conn.commit()
                    conn.close()
                    
            except Exception as e:
                logger.error(f"❌ خطأ في معالجة المسلسل {series.get('id', 'unknown')}: {e}")
                continue

        total_new_items += page_new_items
        total_updated_items += page_updated_items
        
        logger.info(f"✅ تم حفظ صفحة المسلسلات {page_number + 1} بنجاح (+{page_new_items} جديد, ~{page_updated_items} محدث)")
        
        # تحديث التقدم
        if not is_update:
            update_crawl_progress('series', page_number, total_pages, page_new_items, page_updated_items)
        
        # إضافة تأخير لتجنب حظر الخادم
        time.sleep(2)

    # تحديث حالة الإكمال
    if not is_update:
        update_crawl_progress('series', total_pages-1, total_pages, total_new_items, total_updated_items, 'completed')
    
    logger.info(f"✅ تمت عملية {operation_type} المسلسلات بالكامل")
    logger.info(f"📊 الإحصائيات النهائية: {total_new_items} جديد, {total_updated_items} محدث")

def fetch_and_store_movie_data(total_pages, is_update=False):
    """جلب وتخزين بيانات الأفلام"""
    operation_type = "تحديث" if is_update else "جلب"
    logger.info(f"🎬 بدء {operation_type} {total_pages} صفحة من الأفلام...")
    
    total_new_items = 0
    total_updated_items = 0
    
    # تحديد نطاق الصفحات للجلب
    if is_update:
        # في وضع التحديث: نفحص فقط أحدث الصفحات
        start_page = max(0, 599 - total_pages)  # افتراض أن 599 هو أقصى صفحة
        end_page = 599
        pages_to_fetch = list(range(start_page, end_page + 1))
        logger.info(f"🔄 فحص أحدث {total_pages} صفحة أفلام: من {start_page} إلى {end_page}")
    else:
        # في وضع الجلب الكامل: كل الصفحات من الأحدث إلى الأقدم
        pages_to_fetch = list(range(total_pages - 1, -1, -1))
        logger.info(f"📥 جلب كامل لـ {total_pages} صفحة أفلام (من الأحدث إلى الأقدم)")
    
    for page_number in pages_to_fetch:
        logger.info(f"🎥 جلب صفحة الأفلام {page_number + 1}")
        
        try:
            url = f'https://app.arabypros.com/api/poster/by/filtres/1/created/{page_number}/4F5A9C3D9A86FA54EACEDDD635185/d506abfd-9fe2-4b71-b979-feff21bcad13/'
            response = requests.get(url, timeout=30)
            response.raise_for_status()
            movie_data = response.json()
            
            # حفظ البيانات في JSON مع دعم UTF-8
            save_to_json('movies', movie_data, page_number)
            
        except requests.RequestException as e:
            logger.error(f"❌ خطأ في جلب صفحة الأفلام {page_number}: {e}")
            continue

        page_new_items = 0
        page_updated_items = 0
        
        for movie in movie_data:
            try:
                # استخدام Lock لمنع مشكلة database locked
                with db_lock:
                    conn = sqlite3.connect(DB_PATH)
                    cursor = conn.cursor()
                    
                    # حساب هاش البيانات لاكتشاف التغييرات
                    movie_hash = calculate_data_hash(movie)
                    
                    # التحقق مما إذا كانت البيانات موجودة ومتطابقة
                    cursor.execute('SELECT data_hash, is_new FROM movies WHERE id = ?', (movie['id'],))
                    existing_data = cursor.fetchone()
                    
                    if existing_data:
                        existing_hash, is_new = existing_data
                        if existing_hash == movie_hash:
                            # البيانات لم تتغير
                            conn.close()
                            continue
                        else:
                            # البيانات تغيرت - تحديث
                            cursor.execute('''
                                UPDATE movies SET 
                                title=?, description=?, year=?, imdb=?, rating=?, duration=?, 
                                classification=?, image=?, cover=?, trailer_url=?, page_number=?, 
                                data_hash=?, last_updated=CURRENT_TIMESTAMP, is_new=0
                                WHERE id=?
                            ''', (
                                movie['title'], movie['description'], movie['year'],
                                movie['imdb'], movie['rating'], movie['duration'],
                                movie['classification'], movie['image'], movie.get('cover'),
                                movie.get('trailer', {}).get('url'), page_number, movie_hash, movie['id']
                            ))
                            page_updated_items += 1
                            logger.info(f"🔄 تم تحديث الفيلم: {movie['title']}")
                    else:
                        # بيانات جديدة - إدخال
                        cursor.execute('''
                            INSERT INTO movies 
                            (id, title, description, year, imdb, rating, duration, classification, 
                             image, cover, trailer_url, page_number, data_hash, is_new)
                            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1)
                        ''', (
                            movie['id'], movie['title'], movie['description'], movie['year'],
                            movie['imdb'], movie['rating'], movie['duration'], movie['classification'],
                            movie['image'], movie.get('cover'), movie.get('trailer', {}).get('url'),
                            page_number, movie_hash
                        ))
                        page_new_items += 1
                        logger.info(f"✅ تم إضافة فيلم جديد: {movie['title']}")
                    
                    # جلب روابط التشغيل للأفلام
                    try:
                        sources_url = f'https://app.arabypros.com/api/movie/source/by/{movie["id"]}/4F5A9C3D9A86FA54EACEDDD635185/d506abfd-9fe2-4b71-b979-feff21bcad13/'
                        sources_response = requests.get(sources_url, timeout=30)
                        sources_response.raise_for_status()
                        sources = sources_response.json()
                        logger.info(f"🎬 تم جلب {len(sources)} رابط تشغيل للفيلم {movie['id']}")
                    except requests.RequestException as e:
                        logger.error(f"❌ خطأ في جلب روابط الفيلم {movie['id']}: {e}")
                        sources = []

                    for source_index, source in enumerate(sources):
                        source_hash = calculate_data_hash(source)
                        cursor.execute('''
                            INSERT OR REPLACE INTO movie_sources 
                            (id, movie_id, title, url, quality, source_order, data_hash, last_updated)
                            VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
                        ''', (
                            source['id'],
                            movie['id'],
                            source['title'],
                            source['url'],
                            source['quality'],
                            source_index,
                            source_hash
                        ))

                    conn.commit()
                    conn.close()
                    
            except Exception as e:
                logger.error(f"❌ خطأ في معالجة الفيلم {movie.get('id', 'unknown')}: {e}")
                continue

        total_new_items += page_new_items
        total_updated_items += page_updated_items
        
        logger.info(f"✅ تم حفظ صفحة الأفلام {page_number + 1} بنجاح (+{page_new_items} جديد, ~{page_updated_items} محدث)")
        
        # تحديث التقدم
        if not is_update:
            update_crawl_progress('movies', page_number, total_pages, page_new_items, page_updated_items)
        
        # إضافة تأخير لتجنب حظر الخادم
        time.sleep(2)

    # تحديث حالة الإكمال
    if not is_update:
        update_crawl_progress('movies', total_pages-1, total_pages, total_new_items, total_updated_items, 'completed')
    
    logger.info(f"✅ تمت عملية {operation_type} الأفلام بالكامل")
    logger.info(f"📊 الإحصائيات النهائية: {total_new_items} جديد, {total_updated_items} محدث")

def run_in_thread(func, *args):
    """تشغيل الدالة في thread منفصل"""
    thread = Thread(target=func, args=args)
    thread.daemon = True
    thread.start()

# ========== Routes ==========

@app.route('/start', methods=['GET'])
def start_fetching():
    """بدء عملية الجلب الكاملة (لأول مرة)"""
    movie_pages = int(request.args.get('movie_pages', 599))
    series_pages = int(request.args.get('series_pages', 84))
    
    logger.info(f"🚀 بدء الجلب الكامل: {movie_pages} صفحة أفلام، {series_pages} صفحة مسلسلات")
    
    # تشغيل عمليات الجلب في threads منفصلة
    run_in_thread(fetch_and_store_series_data, series_pages, False)
    run_in_thread(fetch_and_store_movie_data, movie_pages, False)
    
    return jsonify({
        "status": "تم بدء الجلب الكامل للبيانات التاريخية",
        "movie_pages": movie_pages,
        "series_pages": series_pages,
        "note": "البيانات تُجلب من الأحدث إلى الأقدم. استخدم /update للتحديثات الدورية لاحقاً."
    }), 200

@app.route('/update', methods=['GET'])
def start_update():
    """بدء عملية التحديثات الدورية (للاستخدام اليومي)"""
    pages_to_check = int(request.args.get('pages', 10))
    
    logger.info(f"🔄 بدء التحديث الدوري لأحدث {pages_to_check} صفحة")
    
    # تشغيل عمليات التحديث في threads منفصلة
    run_in_thread(fetch_and_store_series_data, pages_to_check, True)
    run_in_thread(fetch_and_store_movie_data, pages_to_check, True)
    
    return jsonify({
        "status": "تم بدء عملية التحديث الدوري",
        "pages_checked": pages_to_check,
        "note": f"يتم فحص أحدث {pages_to_check} صفحة فقط للبحث عن التحديثات والإضافات الجديدة"
    }), 200

@app.route('/progress', methods=['GET'])
def get_progress():
    """الحصول على تقدم عمليات الجلب"""
    try:
        with db_lock:
            conn = sqlite3.connect(DB_PATH)
            cursor = conn.cursor()
            
            cursor.execute('''
                SELECT data_type, total_pages, completed_pages, status, 
                       items_added, items_updated, start_time, end_time
                FROM crawl_progress 
                ORDER BY start_time DESC 
                LIMIT 10
            ''')
            
            progress_data = cursor.fetchall()
            conn.close()
            
            result = []
            for row in progress_data:
                data_type, total_pages, completed_pages, status, items_added, items_updated, start_time, end_time = row
                
                progress_percent = (completed_pages / total_pages * 100) if total_pages > 0 else 0
                
                result.append({
                    "data_type": data_type,
                    "total_pages": total_pages,
                    "completed_pages": completed_pages,
                    "progress_percent": round(progress_percent, 1),
                    "status": status,
                    "items_added": items_added,
                    "items_updated": items_updated,
                    "start_time": start_time,
                    "end_time": end_time,
                    "duration": str(datetime.fromisoformat(end_time) - datetime.fromisoformat(start_time)) if end_time and start_time else None
                })
            
            return jsonify({"progress": result}), 200
            
    except Exception as e:
        return jsonify({"error": str(e)}), 500

@app.route('/new_content', methods=['GET'])
def get_new_content():
    """الحصول على المحتوى المضاف حديثاً"""
    try:
        with db_lock:
            conn = sqlite3.connect(DB_PATH)
            cursor = conn.cursor()
            
            # المسلسلات الجديدة
            cursor.execute('''
                SELECT id, title, page_number, created_at 
                FROM series 
                WHERE is_new = 1 
                ORDER BY created_at DESC 
                LIMIT 20
            ''')
            new_series = cursor.fetchall()
            
            # الأفلام الجديدة
            cursor.execute('''
                SELECT id, title, page_number, created_at 
                FROM movies 
                WHERE is_new = 1 
                ORDER BY created_at DESC 
                LIMIT 20
            ''')
            new_movies = cursor.fetchall()
            
            conn.close()
            
            return jsonify({
                "new_series": [
                    {"id": s[0], "title": s[1], "page": s[2], "added": s[3]} 
                    for s in new_series
                ],
                "new_movies": [
                    {"id": m[0], "title": m[1], "page": m[2], "added": m[3]} 
                    for m in new_movies
                ]
            }), 200
            
    except Exception as e:
        return jsonify({"error": str(e)}), 500

@app.route('/status', methods=['GET'])
def get_status():
    """الحصول على حالة قاعدة البيانات"""
    try:
        with db_lock:
            conn = sqlite3.connect(DB_PATH)
            cursor = conn.cursor()
            
            # عدادات البيانات
            cursor.execute("SELECT COUNT(*) FROM series")
            series_count = cursor.fetchone()[0]
            
            cursor.execute("SELECT COUNT(*) FROM seasons")
            seasons_count = cursor.fetchone()[0]
            
            cursor.execute("SELECT COUNT(*) FROM episodes")
            episodes_count = cursor.fetchone()[0]
            
            cursor.execute("SELECT COUNT(*) FROM episode_sources")
            episode_sources_count = cursor.fetchone()[0]
            
            cursor.execute("SELECT COUNT(*) FROM movies")
            movies_count = cursor.fetchone()[0]
            
            cursor.execute("SELECT COUNT(*) FROM movie_sources")
            movie_sources_count = cursor.fetchone()[0]
            
            # أحدث المسلسلات والأفلام
            cursor.execute('''
                SELECT id, title, page_number, created_at 
                FROM series 
                ORDER BY created_at DESC 
                LIMIT 5
            ''')
            latest_series = cursor.fetchall()
            
            cursor.execute('''
                SELECT id, title, page_number, created_at 
                FROM movies 
                ORDER BY created_at DESC 
                LIMIT 5
            ''')
            latest_movies = cursor.fetchall()
            
            conn.close()
            
            return jsonify({
                "database": {
                    "size_mb": round(os.path.getsize(DB_PATH) / (1024 * 1024), 2),
                    "path": DB_PATH
                },
                "counts": {
                    "series": series_count,
                    "seasons": seasons_count,
                    "episodes": episodes_count,
                    "episode_sources": episode_sources_count,
                    "movies": movies_count,
                    "movie_sources": movie_sources_count
                },
                "latest_series": [
                    {"id": s[0], "title": s[1], "page": s[2], "added": s[3]} 
                    for s in latest_series
                ],
                "latest_movies": [
                    {"id": m[0], "title": m[1], "page": m[2], "added": m[3]} 
                    for m in latest_movies
                ]
            }), 200
            
    except Exception as e:
        return jsonify({"error": str(e)}), 500

@app.route('/series/latest', methods=['GET'])
def get_latest_series():
    """الحصول على أحدث المسلسلات (من الأحدث إلى الأقدم)"""
    try:
        with db_lock:
            conn = sqlite3.connect(DB_PATH)
            cursor = conn.cursor()
            
            page = int(request.args.get('page', 1))
            per_page = int(request.args.get('per_page', 20))
            offset = (page - 1) * per_page
            
            cursor.execute('''
                SELECT id, title, description, year, imdb, rating, duration, 
                       classification, image, cover, trailer_url, page_number, created_at
                FROM series 
                ORDER BY created_at DESC, page_number DESC
                LIMIT ? OFFSET ?
            ''', (per_page, offset))
            
            series = cursor.fetchall()
            conn.close()
            
            result = []
            for s in series:
                result.append({
                    "id": s[0],
                    "title": s[1],
                    "description": s[2],
                    "year": s[3],
                    "imdb": s[4],
                    "rating": s[5],
                    "duration": s[6],
                    "classification": s[7],
                    "image": s[8],
                    "cover": s[9],
                    "trailer_url": s[10],
                    "page_number": s[11],
                    "created_at": s[12]
                })
            
            return jsonify({
                "page": page,
                "per_page": per_page,
                "total_results": len(result),
                "series": result
            }), 200
            
    except Exception as e:
        return jsonify({"error": str(e)}), 500

@app.route('/movies/latest', methods=['GET'])
def get_latest_movies():
    """الحصول على أحدث الأفلام (من الأحدث إلى الأقدم)"""
    try:
        with db_lock:
            conn = sqlite3.connect(DB_PATH)
            cursor = conn.cursor()
            
            page = int(request.args.get('page', 1))
            per_page = int(request.args.get('per_page', 20))
            offset = (page - 1) * per_page
            
            cursor.execute('''
                SELECT id, title, description, year, imdb, rating, duration, 
                       classification, image, cover, trailer_url, page_number, created_at
                FROM movies 
                ORDER BY created_at DESC, page_number DESC
                LIMIT ? OFFSET ?
            ''', (per_page, offset))
            
            movies = cursor.fetchall()
            conn.close()
            
            result = []
            for m in movies:
                result.append({
                    "id": m[0],
                    "title": m[1],
                    "description": m[2],
                    "year": m[3],
                    "imdb": m[4],
                    "rating": m[5],
                    "duration": m[6],
                    "classification": m[7],
                    "image": m[8],
                    "cover": m[9],
                    "trailer_url": m[10],
                    "page_number": m[11],
                    "created_at": m[12]
                })
            
            return jsonify({
                "page": page,
                "per_page": per_page,
                "total_results": len(result),
                "movies": result
            }), 200
            
    except Exception as e:
        return jsonify({"error": str(e)}), 500

@app.route('/json/files', methods=['GET'])
def list_json_files():
    """عرض قائمة ملفات JSON المحفوظة"""
    try:
        files = []
        for filename in os.listdir(JSON_DIR):
            if filename.endswith('.json'):
                filepath = os.path.join(JSON_DIR, filename)
                files.append({
                    "name": filename,
                    "size_mb": round(os.path.getsize(filepath) / (1024 * 1024), 2),
                    "modified": datetime.fromtimestamp(os.path.getmtime(filepath)).isoformat()
                })
        
        return jsonify({"json_files": files}), 200
    except Exception as e:
        return jsonify({"error": str(e)}), 500

if __name__ == '__main__':
    init_directories()
    init_database()
    logger.info("🚀 تم تشغيل السكربت الذكي النهائي بنجاح!")
    logger.info("✅ دعم UTF-8 الكامل للغة العربية مفعل")
    logger.info("✅ إصلاح مشكلة database locked")
    logger.info("📋 طرق الاستخدام:")
    logger.info("   الجلب الكامل (أول مرة): GET /start?movie_pages=599&series_pages=84")
    logger.info("   التحديث الدوري (كرون):  GET /update?pages=10")
    logger.info("   متابعة التقدم:           GET /progress")
    logger.info("   المحتوى الجديد:         GET /new_content")
    
    app.run(debug=False, host='0.0.0.0', port=5000)