import json
import os
import re

def safe_filename(title):
    """
    Replaces any invalid characters in the title string with underscores
    to ensure the filename is safe for use in the filesystem.
    """
    invalid_chars = r'[<>:"/\\|?*\x00-\x1f\x80-\xff]'
    safe_title = re.sub(invalid_chars, '_', title).strip()
    return safe_title if safe_title else "Untitled"

def extract_filename_from_url(url):
    """
    Extracts a filename from a URL by removing the protocol, domain, and special characters.
    """
    path = re.sub(r'https?://[^/]+/', '', url)
    path = re.sub(r'[?#].*', '', path)
    path = path.replace('/', '_')
    return safe_filename(path)

def read_existing_titles(titles_file):
    """
    Reads the existing titles from the 'titles_file' if it exists
    and returns them as a set to avoid duplicates.
    """
    if os.path.exists(titles_file):
        with open(titles_file, 'r', encoding='utf8') as file:
            return set(file.read().splitlines())
    return set()

def create_folder_for_input_file(input_file):
    """
    Creates a folder named after the input file without its extension
    if it doesn't already exist.
    """
    folder_name = os.path.splitext(input_file)[0]
    if not os.path.exists(folder_name):
        os.makedirs(folder_name)
    return folder_name

def split_file(input_file, titles_file='titles_list.txt'):
    """
    Reads the 'input_file', splits its content by JSON objects, and processes each entry.
    Saves each entry as a separate JSON file within a folder named after the input file.
    If 'titles_file' exists, it reads and updates it with new titles to avoid duplicates.
    """
    # Read existing titles to avoid duplicates
    existing_titles = read_existing_titles(titles_file)
    
    # Create a folder to save output files
    folder_name = create_folder_for_input_file(input_file)
    
    # Read the input JSON file
    with open(input_file, 'r', encoding='utf8') as file:
        content = file.read()

    # Try to parse JSON as a list
    try:
        data_list = json.loads(content)
        if not isinstance(data_list, list):
            raise ValueError("JSON content is not a list.")
    except json.JSONDecodeError as e:
        print(f"Error parsing JSON as a list: {e}")
        # If parsing fails, attempt manual splitting and parsing of entries
        entries = content.split('},')
        data_list = []
        for entry in entries:
            entry = entry.strip()
            if not entry.endswith('}'):
                entry += '}'
            try:
                data = json.loads(entry)
                data_list.append(data)
            except json.JSONDecodeError as inner_e:
                print(f"Skipping invalid entry: {entry[:30]}... Error: {inner_e}")
                continue

    # Track new titles to update the titles list file
    new_titles = []

    # Process each JSON entry
    for data in data_list:
        if not isinstance(data, dict):
            print(f"Skipping non-dictionary entry: {data}")
            continue
        
        # Determine the title or fallback to extracting from the URL
        title = data.get("title", "").strip()
        if not title:
            title = extract_filename_from_url(data.get("url", ""))
        else:
            title = safe_filename(title)

        # Create a unique filename for each entry
        filename = os.path.join(folder_name, f"{title}.json")
        counter = 1
        while os.path.exists(filename):
            filename = os.path.join(folder_name, f"{title}_{counter}.json")
            counter += 1

        # If the title is not in the existing titles, process it
        if title not in existing_titles:
            new_titles.append(title)
            try:
                with open(filename, 'w', encoding='utf8') as new_file:
                    json.dump(data, new_file, ensure_ascii=False, indent=4)
            except OSError as file_error:
                print(f"Error writing file {filename}: {file_error}")
                continue

    # Append any new titles to the titles file
    if new_titles:
        try:
            with open(titles_file, 'a', encoding='utf8') as file:
                for title in new_titles:
                    file.write(f"{title}\n")
        except OSError as e:
            print(f"Error writing titles file: {e}")

# Call the split_file function with the specified input file
split_file('output-1.json')  # Replace with your actual file name
    

import os
import json

def create_output_directory(directory):
    """
    Creates an output directory if it doesn't already exist.

    Args:
        directory (str): The path to the output directory.
    """
    if not os.path.exists(directory):
        os.makedirs(directory)
        print(f"Created directory: {directory}")
    else:
        print(f"Directory already exists: {directory}")

def combine_json_files_in_current_directory():
    """
    Combines all JSON files in the current directory into a single JSON file 
    within a newly created output directory.
    """
    # Get the current directory where the script is running
    current_directory = os.path.dirname(os.path.abspath(__file__))

    # Initialize a list to hold combined data
    combined_data = []

    # Loop through all files in the current directory
    for filename in os.listdir(current_directory):
        if filename.endswith(".json"):
            file_path = os.path.join(current_directory, filename)
            with open(file_path, 'r', encoding='utf-8') as f:
                try:
                    data = json.load(f)
                    combined_data.append(data)
                except json.JSONDecodeError as e:
                    print(f"Error reading {file_path}: {e}")

    # Create an output directory in the same location as the script
    output_directory = os.path.join(current_directory, 'combined_output')
    create_output_directory(output_directory)

    # Define the output file path
    output_file_path = os.path.join(output_directory, "combined_output.json")

    # Write the combined data to the output file
    with open(output_file_path, 'w', encoding='utf-8') as f:
        json.dump(combined_data, f, ensure_ascii=False, indent=4)

    print(f"Combined {len(combined_data)} JSON files into {output_file_path}")

# Call the function to combine JSON files in the current directory
combine_json_files_in_current_directory()
    

import json
import os

def process_combined_output():
    """
    Reads combined_output.json, removes the 'html' key from each entry,
    writes a list of titles to titles_list.txt, and writes the cleaned JSON data
    to combined_output.txt.
    """
    # Determine the directory where this script is located
    current_dir = os.path.dirname(os.path.abspath(__file__))
    input_file_path = os.path.join(current_dir, "combined_output.json")
    
    # Check if the input file exists
    if not os.path.exists(input_file_path):
        print(f"Input file '{input_file_path}' not found.")
        return

    # Load JSON data from combined_output.json
    try:
        with open(input_file_path, "r", encoding="utf8") as infile:
            data = json.load(infile)
    except json.JSONDecodeError as e:
        print(f"Error loading JSON from '{input_file_path}': {e}")
        return

    titles = []
    cleaned_entries = []

    # Process each entry in the JSON data
    for entry in data:
        # Remove the 'html' key if it exists
        if "html" in entry:
            entry.pop("html", None)
        
        # Extract the title (defaulting to "Untitled" if missing or empty)
        title = entry.get("title", "").strip() or "Untitled"
        titles.append(title)
        cleaned_entries.append(entry)

    # Write the list of titles to titles_list.txt (one title per line)
    titles_file_path = os.path.join(current_dir, "titles_list.txt")
    try:
        with open(titles_file_path, "w", encoding="utf8") as titles_file:
            for title in titles:
                titles_file.write(title + "\n")
    except OSError as e:
        print(f"Error writing '{titles_file_path}': {e}")
        return

    # Write the cleaned JSON data to combined_output.txt
    output_file_path = os.path.join(current_dir, "combined_output.txt")
    try:
        with open(output_file_path, "w", encoding="utf8") as output_file:
            json.dump(cleaned_entries, output_file, ensure_ascii=False, indent=4)
    except OSError as e:
        print(f"Error writing '{output_file_path}': {e}")
        return

    print("Processing complete.")
    print(f"Titles saved to: {titles_file_path}")
    print(f"Cleaned JSON data saved to: {output_file_path}")

if __name__ == "__main__":
    process_combined_output()