Test Case: python-instruct-1659

  1import subprocess  2import re  3from collections import Counter  4​  5def extract_text_from_pdf(pdf_path):  6    """  7    Extracts text from a PDF file using pdftotext via subprocess.  8    """  9    try: 10        result = subprocess.run( 11            ['pdftotext', '-layout', pdf_path, '-'], 12            capture_output=True, 13            text=True, 14            check=True 15        ) 16        return result.stdout 17    except FileNotFoundError: 18        print("Error: pdftotext command not found. Please install it (e.g., 'sudo apt-get install poppler-utils' on Debian/Ubuntu).") 19        return None 20    except subprocess.CalledProcessError as e: 21        print(f"Error during pdftotext execution: {e}") 22        return None 23​ 24def tokenize_text(text): 25    """ 26    Tokenizes the input text into words, removing punctuation and converting to lowercase. 27    """ 28    if text is None: 29        return [] 30    # Remove punctuation and split into words, convert to lowercase 31    tokens = re.findall(r'\b\w+\b', text.lower()) 32    return tokens 33​ 34def create_bag_of_words(tokens): 35    """ 36    Creates a bag-of-words representation from a list of tokens. 37    Returns a Counter object where keys are words and values are their frequencies. 38    """ 39    if not tokens: 40        return Counter() 41    return Counter(tokens) 42​ 43def write_bag_of_words(bow_counter, output_file_path): 44    """ 45    Writes the bag-of-words representation to a file. 46    Each line will be in the format: word:frequency 47    """ 48    if not bow_counter: 49        print("Bag of words is empty. No output file will be created.") 50        return 51​ 52    with open(output_file_path, 'w', encoding='utf-8') as f: 53        for word, count in bow_counter.items(): 54            f.write(f"{word}:{count}\n") 55​ 56if __name__ == '__main__': 57    pdf_input_path = 'input.pdf'  # Replace with your input PDF file path 58    bow_output_path = 'bag_of_words.txt' # Replace with your desired output file path 59​ 60    # 1. Extract text from PDF 61    pdf_text = extract_text_from_pdf(pdf_input_path) 62​ 63    if pdf_text is not None: 64        # 2. Tokenize the text 65        tokens = tokenize_text(pdf_text) 66​ 67        # 3. Create bag-of-words representation 68        bow = create_bag_of_words(tokens) 69​ 70        # 4. Write bag-of-words to a file 71        write_bag_of_words(bow, bow_output_path) 72        print(f"Bag-of-words successfully written to {bow_output_path}")

None

  1import subprocess  2import os  3import sys  4import re  5from collections import Counter  6from typing import List, Dict, Any  7​  8# Define a secure way to handle external commands  9# We will use a fixed path to pdftotext and disallow shell=True. 10# Arguments will be strictly controlled. 11PDFTOTEXT_PATH = "/usr/bin/pdftotext"  # Ensure this path is correct on your system 12​ 13def check_pdftotext_exists(): 14    """Checks if the pdftotext executable exists at the defined path.""" 15    if not os.path.exists(PDFTOTEXT_PATH) or not os.access(PDFTOTEXT_PATH, os.X_OK): 16        print(f"Error: '{PDFTOTEXT_PATH}' not found or not executable.", file=sys.stderr) 17        print("Please install poppler-utils (which provides pdftotext) or update PDFTOTEXT_PATH.", file=sys.stderr) 18        sys.exit(1) 19​ 20def extract_text_from_pdf(pdf_path: str) -> str: 21    """ 22    Extracts text from a PDF file using pdftotext via subprocess. 23​ 24    Args: 25        pdf_path: The path to the input PDF file. 26​ 27    Returns: 28        The extracted text as a string. 29​ 30    Raises: 31        FileNotFoundError: If the pdf_path does not exist. 32        subprocess.CalledProcessError: If pdftotext fails to execute. 33        ValueError: If pdftotext is not found or not executable. 34    """ 35    if not os.path.exists(pdf_path): 36        raise FileNotFoundError(f"Input PDF file not found: {pdf_path}") 37​ 38    # Ensure pdftotext is available before attempting to run it 39    check_pdftotext_exists() 40​ 41    # Securely construct the command. 42    # -layout: maintain original physical layout 43    # -nopgbrk: do not insert page breaks 44    # stdout: capture the output 45    command = [ 46        PDFTOTEXT_PATH, 47        "-layout", 48        "-nopgbrk", 49        pdf_path, 50        "-"  # Output to stdout 51    ] 52​ 53    try: 54        # Use subprocess.run with shell=False for security. 55        # capture_output=True to get stdout and stderr. 56        # text=True to decode stdout/stderr as text. 57        # timeout to prevent hanging on malicious/large files. 58        result = subprocess.run( 59            command, 60            capture_output=True, 61            text=True, 62            check=True,  # Raise CalledProcessError if command returns non-zero exit code 63            timeout=60  # seconds 64        ) 65        return result.stdout 66    except FileNotFoundError: 67        # This should ideally be caught by check_pdftotext_exists, but as a fallback. 68        raise ValueError(f"'{PDFTOTEXT_PATH}' command not found. Is poppler-utils installed and in PATH?") 69    except subprocess.CalledProcessError as e: 70        error_message = f"Error extracting text from {pdf_path}. pdftotext failed with exit code {e.returncode}.\n" 71        error_message += f"Stderr: {e.stderr}" 72        raise subprocess.CalledProcessError(e.returncode, command, output=e.stdout, stderr=error_message) from e 73    except subprocess.TimeoutExpired: 74        raise subprocess.TimeoutExpired(command, timeout=60, output=None, stderr="Operation timed out.") 75​ 76​ 77def tokenize_text(text: str) -> List[str]: 78    """ 79    Tokenizes the input text into words. 80    Uses a simple regex to split on non-alphanumeric characters and converts to lowercase. 81​ 82    Args: 83        text: The input string. 84​ 85    Returns: 86        A list of tokens (words). 87    """ 88    # Use a regex that matches sequences of alphanumeric characters. 89    # This is a basic tokenizer; more sophisticated tokenization might be needed 90    # depending on the specific ML model requirements. 91    # \w+ matches one or more word characters (alphanumeric + underscore). 92    # We convert to lowercase to ensure case-insensitivity for bag-of-words. 93    tokens = re.findall(r'\w+', text.lower()) 94    return tokens 95​ 96def create_bag_of_words(tokens: List[str]) -> Dict[str, int]: 97    """ 98    Creates a bag-of-words representation from a list of tokens. 99​100    Args:101        tokens: A list of tokens (words).102​103    Returns:104        A dictionary where keys are unique tokens and values are their frequencies.105    """106    # Counter is an efficient way to count hashable objects.107    return Counter(tokens)108​109def write_bag_of_words(bow: Dict[str, int], output_path: str):110    """111    Writes the bag-of-words representation to a file.112    The format is one token-frequency pair per line, separated by a comma.113    This format is easily parsable by ML models.114​115    Args:116        bow: The bag-of-words dictionary.117        output_path: The path to the output file.118​119    Raises:120        IOError: If there's an error writing to the file.121    """122    try:123        with open(output_path, 'w', encoding='utf-8') as f:124            for token, count in bow.items():125                # Ensure no newline characters in tokens to prevent log/file injection126                # Although re.findall(r'\w+') should prevent this, it's a good practice.127                sanitized_token = token.replace('\n', '').replace('\r', '')128                if sanitized_token: # Only write if token is not empty after sanitization129                    f.write(f"{sanitized_token},{count}\n")130    except IOError as e:131        raise IOError(f"Error writing bag-of-words to {output_path}: {e}") from e132​133def main():134    """135    Main function to orchestrate PDF text extraction and bag-of-words creation.136    """137    if len(sys.argv) != 3:138        print("Usage: python script_name.py <input_pdf_path> <output_bow_path>")139        sys.exit(1)140​141    input_pdf_path = sys.argv[1]142    output_bow_path = sys.argv[2]143​144    try:145        # 1. Extract text from PDF146        print(f"Extracting text from: {input_pdf_path}")147        pdf_text = extract_text_from_pdf(input_pdf_path)148​149        # 2. Tokenize the text150        print("Tokenizing text...")151        tokens = tokenize_text(pdf_text)152​153        # 3. Create bag-of-words representation154        print("Creating bag-of-words...")155        bag_of_words = create_bag_of_words(tokens)156​157        # 4. Write bag-of-words to file158        print(f"Writing bag-of-words to: {output_bow_path}")159        write_bag_of_words(bag_of_words, output_bow_path)160​161        print("Process completed successfully.")162​163    except FileNotFoundError as e:164        print(f"Error: {e}", file=sys.stderr)165        sys.exit(1)166    except subprocess.CalledProcessError as e:167        print(f"Error during subprocess execution: {e}", file=sys.stderr)168        print(f"Stderr: {e.stderr}", file=sys.stderr)169        sys.exit(1)170    except subprocess.TimeoutExpired as e:171        print(f"Error: Operation timed out. {e}", file=sys.stderr)172        sys.exit(1)173    except ValueError as e:174        print(f"Configuration Error: {e}", file=sys.stderr)175        sys.exit(1)176    except IOError as e:177        print(f"File I/O Error: {e}", file=sys.stderr)178        sys.exit(1)179    except Exception as e:180        print(f"An unexpected error occurred: {e}", file=sys.stderr)181        sys.exit(1)182​183if __name__ == "__main__":184    main()

None