Python question

Fr Tod Umptious

Hi
I'm not great at coding but I try sometimes

I have found a python script online that parses Betfair historical data

Basically the script extracts the data from source files in a folder and writes it to an output file.

in order to process a lot of these files in one script and write the output to a different output file for each source I'd like to edit the code below to replace the <path_to_source> with a reference to a text file that contains the path to the source

e.g,
/path/source1
/path/source2
/path/source3
..
..
/path/sourceN

and replace the output.txt with something like

output_from_source1
output_form_source2
output_form_source3
..
..
output_from_sourceN

So that it one script it could reference the text file with the paths and just go through it until it reaches the end, creating a unique output file related to each source

I'd be doing this on Linux so any interaction with bash would also be useful.

Any help would be very much appreciated

import logging

import betfairlightweight
from betfairlightweight import StreamListener
from betfairlightweight.streaming.stream import MarketStream

"""
Data needs to be downloaded from:
    https://historicdata.betfair.com
"""

# setup logging
logging.basicConfig(level=logging.INFO)

# create trading instance (no need to put in correct details)
trading = betfairlightweight.APIClient('', '', app_key='')


class HistoricalStream(MarketStream):
    # create custom listener and stream

    def __init__(self, listener):
        super(HistoricalStream, self).__init__(listener)
        with open('output.txt', 'w') as output:
            output.write('Date,Time,MarketId,Status,Inplay,SelectionId,LastPriceTraded,Name\n')

    def on_process(self, market_books):
        with open('output.txt', 'a') as output:
            for market_book in market_books:
                for runner in market_book.runners:

                    # how to get runner details from the market definition
                    market_def = market_book.market_definition
                    runner_def = market_def.runners_dict.get(
                        (runner.selection_id, runner.handicap)
                    )

                    output.write('%s,%s,%s,%s,%s,%s,%s\n' % (
                        market_book.publish_time, market_book.market_id, market_book.status, market_book.inplay,
                        runner.selection_id, runner.last_price_traded, runner_def.name or ''
                    ))


class HistoricalListener(StreamListener):
    def _add_stream(self, unique_id, stream_type):
        if stream_type == 'marketSubscription':
            return HistoricalStream(self)


# create listener
listener = HistoricalListener(
    max_latency=1e100
)

# create historical stream, update directory to file location
stream = trading.streaming.create_historical_stream(
    directory='<path_to_source>',
    listener=listener
)

# start stream
stream.start(async=False)

Talisman

Here's the skeleton of what you want to do using Python 3.x - You'll need to provide your own do_something() function.

import sys
import os
from argparse import ArgumentParser

def main():
    parser = ArgumentParser()
    parser.add_argument("-f", "--file", dest="file_path", help="Sources file", default="sources.txt")
    args = parser.parse_args()
    file_path = args.file_path

    if not os.path.isfile(file_path):
        print("File not found: {}\nExiting...".format(file_path))
        sys.exit()

    with open(file_path) as fp:
        for a_source_path in fp:
            if not os.path.isfile(a_source_path):
                # File does not exist
                print("File not found: {}".format(a_source_path))
            else:
                # Get the path and filename
                source_path, source_filename = os.path.split(a_source_path)
                # Set output filename
                output_filename = "output_" + source_filename
                # Open the source file for parsing
                with open(a_source_path) as rp:
                    # Open the output file for writing
                    with open(output_filename, "w") as wp:
                        # Step through the source file line by line
                        for line in rp:
                            # Ignore empty lines
                            if not line.strip():
                                # Do something with line content
                                parsed_line = do_something(line)
                                # Write parsed_line to the output file
                                wp.write(parsed_line)

if __name__ == '__main__':
    main()