import pandas as pd
import requests
import time

# Function to fetch all symbols from Binance
def fetch_all_symbols():
    url = "https://api.binance.com/api/v3/exchangeInfo"
    response = requests.get(url)
    data = response.json()
    symbols = [symbol['symbol'] for symbol in data['symbols'] if symbol['status'] == 'TRADING']
    return symbols

# Function to fetch historical data from Binance with pagination
def fetch_historical_data(symbol, interval, start_str, end_str):
    url = "https://api.binance.com/api/v3/klines"
    start_ts = int(pd.to_datetime(start_str).timestamp() * 1000)
    end_ts = int(pd.to_datetime(end_str).timestamp() * 1000)
    all_data = []

    while start_ts < end_ts:
        params = {
            "symbol": symbol,
            "interval": interval,
            "startTime": start_ts,
            "endTime": end_ts,
            "limit": 1000
        }
        response = requests.get(url, params=params)
        data = response.json()

        if len(data) == 0:
            break
        
        # Append the data
        all_data.extend(data)
        print(f"Data count: {len(all_data)}")
        
        # Update the start timestamp for the next iteration
        start_ts = data[-1][0] + 1  # Move to the next millisecond after the last received data point

        # Sleep to respect rate limits
        time.sleep(0.1)

    if len(all_data) == 0:
        return pd.DataFrame()  # Return an empty DataFrame if no data

    # Check the number of columns in the data and adjust accordingly
    if len(all_data[0]) == 12:
        columns = [
            'open_time', 'open', 'high', 'low', 'close', 'volume',
            'close_time', 'quote_asset_volume', 'number_of_trades',
            'taker_buy_base_asset_volume', 'taker_buy_quote_asset_volume', 'ignore'
        ]
    else:
        raise ValueError(f"Unexpected number of columns: {len(all_data[0])}")

    df = pd.DataFrame(all_data, columns=columns)
    df['open_time'] = pd.to_datetime(df['open_time'], unit='ms')
    df['close'] = df['close'].astype(float)
    return df[['open_time', 'close']]

# Define parameters
interval = '1s'  # 1-second intervals
start_str = '2023-05-01'
end_str = '2023-06-30'

# Fetch all trading symbols
symbols = fetch_all_symbols()

# Fetch and save data for each symbol
for symbol in symbols:
    try:
        if (not symbol.endswith('USDT') or symbol == 'BTCUSDT' or symbol == 'ETHUSDT'):
            print(f"Skipping {symbol}")
            continue
        print(f"Fetching data for {symbol}...")
        df = fetch_historical_data(symbol, interval, start_str, end_str)
        if df.empty:
            print(f"No data for {symbol}")
            continue
        csv_file = f"data/{symbol}.csv"
        df.to_csv(csv_file, index=False)
        print(f"Data for {symbol} saved to {csv_file}")
        # break
    except Exception as e:
        print(f"Error fetching data for {symbol}: {e}")