我在开发一个Polymarket机器人，并从Bybit下载了历史比特币数据。每次尝试遍历数据时，我都会遇到这个错误：

json.decoder.JSONDecodeError: Extra data: line 2 column 1 (char 9035)

我到底哪里做错了？

import json
import pytz
import pandas as pd
from datetime import datetime, timezone
import requests
orderbook = "C:/Users/Owner/PycharmProjects/polymarket strategy/.venv/historical orderbook data"
import os
data = []
file_number = 0
class File:
    def __init__(self,file_number):
        self.file_number = file_number
        file_number += 1
    def get_timestamp(self):
        with open(filepath, 'r') as file:
            json.load(file)
            timestamps = []
            for row in file:
                timestamps.append(row.get("ts"))
            print(timestamps)

        lowest_timestamp = timestamps[0]
        highest_timestamp = timestamps[-1]
for filename in os.listdir(orderbook):
    filepath = os.path.join(orderbook, filename)
    data.append(filepath)
    file_number += 1
    filename = File(file_number)
for filepath in data:
    File.get_timestamp(filepath)

要遍历的文件有很多，因此如果可能的话我不想修改这些文件。文件中有多个字典。

文件看起来像这样：

{"topic":"orderbook.200.BTCUSDT","ts":1771718402029,"type":"snapshot","data":{"s":"BTCUSDT","b":[["67977.8","0.482205"],["67976.7","0.000732"],["67975.9","0.01413"],["67973.7","0.007355"],["67971.2","0.007357"],["67971.1","0.023876"],["67971","0.0787"],["67970.2","0.000572"],["67969.1","0.069358"],["67968.9","0.2"],["67968.7","0.000111"],["67968.6","0.2"],["67968.4","0.000498"],["67967.7","0.002"],["67967","0.031345"],["67966.8","0.0004"],["67966","0.088653"],["67964.5","0.088872"],["67964.2","0.000258"],["67963.8","0.0787"],["67963.5","0.0946"],["67963.4","0.047698"],["67963","0.01413"],["67961.9","0.000111"],["67961.6","0.002"],["67959.6","0.007356"],["67958.8","0.000572"],["67958.5","0.007855"],["67957.7","0.00012"],["67957.4","0.005297"],["67955.6","0.102588"],["67955.5","0.0883"],["67955.1","0.000111"],["67955","0.11034"],["67954.8","0.0787"],["67953.6","0.00348"],["67953.5","0.000498"],["67953.4","0.00203"],["67953.1","0.000572"],["67952.5","0.05229"],["67952","0.063133"],["67951.9","0.100649"],["67950.6","0.007358"],["67949.2","0.115197"],["67949","0.000247"],["67948.5","0.000498"],["67948.3","0.000111"],["67947.3","0.000572"],["67947","0.082786"],["67946.9","0.11034"],["67946.8","0.0883"],["67946.6","0.0787"],["67944.7","0.05914"],["67944.6","0.196626"],["67943.5","0.000498"],["67942.4","0.001"],["67941.8","0.007356"],["67941.6","0.000572"],["67941.5","0.000516"],["67941.4","0.06959"],["67940.4","0.007359"],["67940.3","0.00088"],["67940","0.0436"],["67939.9","0.043212"],["67939.6","0.11034"],["67939.5","0.0787"],["67938.6","0.000498"],["67938.5","0.01413"],["67938.2","0.10614"],["67938.1","0.0883"],["67937.6","0.266595"],["67937.1","0.007359"],["67936.5","0.000111"],["67935.9","0.000572"],["67934.7","0.000111"],["67934.4","0.001472"],["67933.9","0.00012"],["67933.6","0.003058"],["67931.9","0.204205"],["67931.8","0.11034"],["67930.2","0.000572"],["67929.1","0.0787"],["67928.9","0.19818"],["67928.6","0.000498"],["67927.9","0.000111"],["67927","0.02676"],["67926.9","0.0883"],["67926.8","0.001347"],["67925","0.102847"],["67924.9","0.176475"],["67924.4","0.000572"],["67923.6","0.000498"],["67922.2","0.0787"],["67922.1","0.0883"],["67921.7","0.001"],["67921.2","0.03855"],["67921.1","0.125168"],["67921","0.014717"],["67920.9","0.014717"],["67920","0.000357"],["67919","0.014718"],["67918.9","0.000551"],["67918.7","0.00107"],["67917.5","0.001398"],["67915.4","0.260667"],["67915.3","0.079251"],["67915","0.814015"],["67914.6","0.0883"],["67914.4","0.198744"],["67914.3","0.01483"],["67914.2","0.014719"],["67914.1","0.212899"],["67913.7","0.000498"],["67913","0.110912"],["67912.8","0.000126"],["67912.1","0.004894"],["67911.2","0.002"],["67910.5","0.0883"],["67910.4","0.201659"],["67910.3","0.596375"],["67908.7","0.000498"],["67908.4","0.0787"],["67907.5","0.000111"],["67907.3","0.000572"],["67907.2","0.026387"],["67907.1","0.0883"],["67905.4","0.0001"],["67904.1","0.11034"],["67904","0.466572"],["67903.9","0.596431"],["67903.7","0.000498"],["67903.1","0.0883"],["67903","0.19818"],["67902.4","0.000111"],["67902","0.000363"],["67901.6","0.5999"],["67901.5","0.079272"],["67900.7","0.000111"],["67900.4","0.001473"],["67900","0.000332"],["67899.7","0.026842"],["67899.6","0.0883"],["67899.4","0.001126"],["67898.8","0.000498"],["67896.1","0.072633"],["67896","0.0883"],["67895.9","0.073401"],["67895.8","0.000572"],["67894.7","0.001914"],["67894.6","0.079139"],["67894.2","0.000278"],["67894","0.000483"],["67893.9","0.35228"],["67893.8","0.702411"],["67893.6","0.072787"],["67893.5","0.006606"],["67892.8","0.000242"],["67892.6","0.000449"],["67892.5","0.000235"],["67892.4","0.197329"],["67892.2","0.11181"],["67892.1","0.0883"],["67891","0.003063"],["67890.6","0.000321"],["67890.5","0.000243"],["67890.1","0.000572"],["67889.3","0.014874"],["67888.8","0.000729"],["67888.7","0.03"],["67888.4","0.002314"],["67888","0.000716"],["67887.7","0.0787"],["67887.5","0.160067"],["67886.8","0.001473"],["67886.5","0.000948"],["67886.1","0.000456"],

这会持续一段时间，直到出现另一份包含类似信息的快照

解决方案

问题在于它不是一个JSON文件，而是一个 multi-JSON 文件。文件的每一行都是一个独立的JSON。

它需要逐行从文件读取，并将每一行转换成 json.loads(text)

    with open(filepath, "r") as file:
        timestamps = []

        for line in file:
            row = json.loads(line)
            timestamps.append(row.get("ts"))

        print(timestamps)

我从 https://www.bybit.com/derivatives/en/history-data 下载了一些数据并进行了测试。
单日的数据解压后大约是40MB，行数大约是190,607行。

用于测试的完整代码：

import os
import json

# --- classes ---


class File:

    def __init__(self, path, number=0):
        self.path = path
        self.number = number

    def get_timestamp(self):
        result = []
        with open(self.path, "r") as file:
            timestamps = []
            for line in file:
                row = json.loads(line)
                timestamps.append(row.get("ts"))
            print(timestamps)

        lowest_timestamp = timestamps[0]
        highest_timestamp = timestamps[-1]

        return lowest_timestamp, highest_timestamp


# --- functions ---

# --- main ---

#orderbook = "C:/Users/Owner/PycharmProjects/polymarket strategy/.venv/historical orderbook data"
orderbook = "./historical orderbook data"

data = []

for number, filename in enumerate(os.listdir(orderbook)):

    print(f"{number:3} | {filename}")

    fullpath = os.path.join(orderbook, filename)
    file = File(fullpath, number)

    data.append(file)

print("--- get_timestamp ---")

for file in data:
    print(file.get_timestamp())