Strings: Immutable Sequences & Text Processing

# Creating strings -- four equivalent ways
s1 = 'single quotes'
s2 = "double quotes"
s3 = '''triple single quotes (can span lines)'''
s4 = """triple double quotes (can span lines)"""

# Strings are sequences -- you can iterate
word = "Python"
for char in word:
    print(char, end=" ")   # P y t h o n

print()
print(len(word))           # 6
print(word[0])             # 'P'
print(word[-1])            # 'n'

# But you CANNOT change a character
try:
    word[0] = "J"          # TypeError!
except TypeError as e:
    print(f"Error: {e}")   # 'str' object does not support item assignment

# Every "modification" returns a NEW string object
name = "alice"
upper = name.upper()         # new object
print(name)                  # "alice"  -- original unchanged!
print(upper)                 # "ALICE"  -- new object

# To actually change the variable, reassign it
name = name.upper()
print(name)                  # "ALICE"

# Strings can be used as dict keys (because they're immutable and hashable)
word_count = {
    "machine": 42,
    "learning": 38,
    "data": 55,
}
print(word_count["machine"])  # 42

s = "Python"
#    0123456   (positive indices)
#   -6-5-4-3-2-1  (negative indices)

# Single character access
print(s[0])    # 'P'  -- first character
print(s[1])    # 'y'
print(s[5])    # 'n'  -- last character (index = len-1)
print(s[-1])   # 'n'  -- last character (negative: from end)
print(s[-2])   # 'o'  -- second to last
print(s[-6])   # 'P'  -- first character via negative index

# Slicing: s[start:stop:step]
# start is inclusive, stop is exclusive

s = "Hello, World!"

print(s[0:5])       # "Hello"    -- index 0 to 4
print(s[7:12])      # "World"    -- index 7 to 11
print(s[:5])        # "Hello"    -- from start to 4
print(s[7:])        # "World!"   -- from 7 to end
print(s[:])         # "Hello, World!" -- full copy

# Step
print(s[::2])       # "Hlo ol!"  -- every other character
print(s[::3])       # "H,rd"     -- every third character
print(s[::-1])      # "!dlroW ,olleH" -- reversed!

# Negative start/stop
print(s[-6:])       # "orld!"    -- last 6 characters
print(s[-6:-1])     # "orld"     -- last 6, excluding very last

# Practical slicing patterns
filename = "report_2024_final.pdf"
print(filename[:-4])           # "report_2024_final" -- strip extension
print(filename[-3:])           # "pdf" -- get extension

email = "alice@example.com"
at_pos = email.index("@")
username = email[:at_pos]
domain = email[at_pos+1:]
print(username)                # "alice"
print(domain)                  # "example.com"

# Palindrome check using reversal
def is_palindrome(s):
    s = s.lower().replace(" ", "")
    return s == s[::-1]

print(is_palindrome("racecar"))       # True
print(is_palindrome("A man a plan a canal Panama"))  # True (after cleanup)
print(is_palindrome("hello"))         # False

text = "hello, World! this is PYTHON."

print(text.upper())       # "HELLO, WORLD! THIS IS PYTHON."
print(text.lower())       # "hello, world! this is python."
print(text.title())       # "Hello, World! This Is Python."
print(text.capitalize())  # "Hello, world! this is python."  (only first char up)
print(text.swapcase())    # "HELLO, wORLD! THIS IS python."

# Case normalization is critical for NLP
words = ["Machine", "LEARNING", "machine", "MACHINE"]
normalized = [w.lower() for w in words]
unique = set(normalized)
print(unique)             # {'machine', 'learning'} -- deduplication works correctly

# These return True or False
print("hello123".isalnum())    # True  -- all alphanumeric
print("hello".isalpha())       # True  -- all letters
print("12345".isdigit())       # True  -- all digits
print("   ".isspace())         # True  -- all whitespace
print("Hello World".istitle()) # True  -- title case

# Starts/ends with
url = "https://api.example.com/v1/models"
print(url.startswith("https://"))  # True
print(url.endswith("/models"))      # True

# Multiple prefixes/suffixes (pass a tuple)
filename = "data.csv"
print(filename.endswith((".csv", ".tsv", ".parquet")))  # True

# Checking substring membership (using 'in' operator)
sentence = "machine learning is fascinating"
print("learning" in sentence)    # True
print("deep" in sentence)        # False
print("learning" not in sentence) # False

text = "the cat sat on the mat"

# find() returns index of first occurrence, -1 if not found (safe)
print(text.find("cat"))      # 4
print(text.find("dog"))      # -1  -- not found, no error

# index() same as find() but raises ValueError if not found
print(text.index("cat"))     # 4
try:
    text.index("dog")
except ValueError as e:
    print(f"Error: {e}")     # substring not found

# rfind() and rindex() search from the RIGHT
print(text.rfind("the"))     # 18  -- last occurrence
print(text.rfind("at"))      # 20  -- last "at"

# count() counts non-overlapping occurrences
print(text.count("the"))     # 2
print(text.count("at"))      # 3  -- "cat", "sat", "mat"
print("aaa".count("aa"))     # 1  -- non-overlapping!

# replace() -- returns new string with substitutions
text = "I love cats. Cats are great. My cat is named Whiskers."
print(text.replace("cat", "dog"))
# "I love dogs. Cats are great. My dog is named Whiskers."
# Note: case-sensitive! "Cats" was not replaced

print(text.replace("cat", "dog", 1))  # replace only first occurrence
# "I love dogs. Cats are great. My cat is named Whiskers."

# strip(), lstrip(), rstrip() -- remove whitespace (or specified chars)
messy = "  \t  hello world  \n  "
print(repr(messy.strip()))         # 'hello world'
print(repr(messy.lstrip()))        # 'hello world  \n  '
print(repr(messy.rstrip()))        # '  \t  hello world'

# Strip specific characters
csv_field = ',"Alice",'
print(csv_field.strip(',').strip('"'))  # 'Alice'

# Data cleaning in NLP
dirty_tokens = ["  hello  ", "\nworld\t", "  python  "]
clean_tokens = [t.strip() for t in dirty_tokens]
print(clean_tokens)  # ['hello', 'world', 'python']

# split() -- split on whitespace by default (handles multiple spaces, tabs, newlines)
sentence = "  the   quick   brown   fox  "
words = sentence.split()
print(words)         # ['the', 'quick', 'brown', 'fox']  -- cleaned up!

# split(delimiter) -- split on specific character
csv = "Alice,25,Engineer,New York"
fields = csv.split(",")
print(fields)        # ['Alice', '25', 'Engineer', 'New York']

# split with maxsplit limit
text = "key=value=with=equals"
key, rest = text.split("=", 1)    # split at most once
print(key)   # "key"
print(rest)  # "value=with=equals"

# rsplit() -- split from the right
path = "/usr/local/bin/python"
print(path.rsplit("/", 1))   # ['/usr/local/bin', 'python']

# splitlines() -- split on any line ending (\n, \r\n, \r)
multiline = "Line 1\nLine 2\r\nLine 3\rLine 4"
print(multiline.splitlines())  # ['Line 1', 'Line 2', 'Line 3', 'Line 4']

# partition() -- split into exactly 3 parts: before, separator, after
url = "https://example.com/path?query=1"
protocol, sep, rest = url.partition("://")
print(protocol)  # "https"
print(rest)      # "example.com/path?query=1"

# join() -- the FAST way to concatenate many strings
words = ["machine", "learning", "is", "fun"]

# join: put separator BETWEEN each item
sentence = " ".join(words)
print(sentence)   # "machine learning is fun"

# Different separators
csv_line = ",".join(words)
print(csv_line)   # "machine,learning,is,fun"

path = "/".join(["usr", "local", "bin", "python"])
print(path)       # "usr/local/bin/python"

# No separator
initials = "".join([w[0].upper() for w in words])
print(initials)   # "MLIF"

# join is the CORRECT way to build strings in a loop (see CommonMistake)
tokens = ["The", "quick", "brown", "fox"]
result = " ".join(tokens)  # ONE allocation, O(n) time
print(result)

# ljust, rjust, center -- pad to a minimum width
name = "Alice"
print(name.ljust(10))          # "Alice     "  -- left-aligned, padded right
print(name.rjust(10))          # "     Alice"  -- right-aligned, padded left
print(name.center(10))         # "  Alice   "  -- centered
print(name.center(10, "-"))    # "--Alice---"  -- custom fill character

# zfill -- pad numbers with leading zeros
for n in [1, 42, 100, 1234]:
    print(str(n).zfill(5))
# 00001
# 00042
# 00100
# 01234

# Practical: formatting a table
headers = ["Name", "Score", "Grade"]
row1 = ["Alice", "95", "A"]
row2 = ["Bob", "78", "C"]

print(f"{headers[0]:<10}{headers[1]:>8}{headers[2]:>8}")
print(f"{row1[0]:<10}{row1[1]:>8}{row1[2]:>8}")
print(f"{row2[0]:<10}{row2[1]:>8}{row2[2]:>8}")
# Name          Score   Grade
# Alice            95       A
# Bob              78       C

# Basic .format(): {} are placeholders, filled by positional args
print("Hello, {}! You are {} years old.".format("Alice", 25))
# Hello, Alice! You are 25 years old.

# Positional index
print("{0} and {1}, then {0} again".format("first", "second"))
# first and second, then first again

# Named placeholders
print("{name} scored {score:.1f}%".format(name="Bob", score=87.3))
# Bob scored 87.3%

# Format spec works the same as f-strings
pi = 3.14159
print("Pi is {:.3f}".format(pi))         # Pi is 3.142
print("Count: {:,}".format(1234567))     # Count: 1,234,567
print("Accuracy: {:.1%}".format(0.934)) # Accuracy: 93.4%

# Filling a template from a dict
template = "Name: {name}, Age: {age}, City: {city}"
data = {"name": "Alice", "age": 30, "city": "London"}
print(template.format(**data))   # Name: Alice, Age: 30, City: London

# Basic f-strings
name = "Alice"
score = 95.678
count = 1234567

print(f"Hello, {name}!")                   # "Hello, Alice!"
print(f"Score: {score}")                   # "Score: 95.678"
print(f"2 + 2 = {2 + 2}")                 # "2 + 2 = 4"  (expressions work!)
print(f"Upper: {name.upper()}")            # "Upper: ALICE"  (method calls work!)

# FLOAT PRECISION: .Nf controls decimal places
pi = 3.14159265358979
print(f"{pi:.2f}")      # "3.14"     -- 2 decimal places
print(f"{pi:.4f}")      # "3.1416"   -- 4 decimal places (rounded)
print(f"{pi:.0f}")      # "3"        -- no decimals (rounded)
print(f"{pi:10.3f}")    # "     3.142" -- width 10, 3 decimals

# THOUSANDS SEPARATOR: , (comma)
big_num = 1234567890
print(f"{big_num:,}")          # "1,234,567,890"
print(f"{big_num:,.2f}")       # "1,234,567,890.00"
print(f"{3.5e6:,.0f}")         # "3,500,000"

# PERCENTAGE: .N% (multiplies by 100, adds %)
accuracy = 0.9734
print(f"{accuracy:.1%}")       # "97.3%"
print(f"{accuracy:.0%}")       # "97%"
print(f"{0.5:.2%}")            # "50.00%"

# ALIGNMENT: < left, > right, ^ center
# {value:fill_char align width}
label = "hello"
print(f"{label:<10}")          # "hello     "  left-aligned, width 10
print(f"{label:>10}")          # "     hello"  right-aligned
print(f"{label:^10}")          # "  hello   "  centered
print(f"{label:*^10}")         # "**hello***"  centered with * fill
print(f"{label:-<10}")         # "hello-----"  left with - fill
print(f"{42:0>8}")             # "00000042"    zero-pad to width 8

# Numbers right-align by default, strings left-align
for name, score in [("Alice", 95), ("Bob", 78), ("Charlie", 88)]:
    print(f"{name:<10} {score:>5}")
# Alice         95
# Bob           78
# Charlie       88

# REPR and STR conversion
data = [1, "hello", None, 3.14]
print(f"{data!r}")    # repr: [1, 'hello', None, 3.14]  (shows quotes)
print(f"{data!s}")    # str:  [1, 'hello', None, 3.14]  (same here)

text = "hello\tworld\n"
print(f"{text!r}")    # 'hello\\tworld\\n'  (shows escape sequences)
print(f"{text!s}")    # hello	world     (interprets escape sequences)

# DEBUG format (Python 3.8+): variable=value
x = 42
y = [1, 2, 3]
name = "Alice"
print(f"{x=}")         # x=42
print(f"{y=}")         # y=[1, 2, 3]
print(f"{name=}")      # name='Alice'
print(f"{x + y[0]=}")  # x + y[0]=43  (expressions too!)

# NESTED f-strings and dynamic formatting
# You can put variables inside the format spec!
width = 10
precision = 3
value = 3.14159

print(f"{value:{width}.{precision}f}")  # "     3.142"

# Dynamic column widths
col_width = 15
data = [("Alice", 95.5), ("Bob", 78.3), ("Charlie", 88.9)]
for name, score in data:
    print(f"{name:{col_width}} {score:.1f}%")

# Integer formats
n = 255
print(f"{n:d}")    # "255"   -- decimal (default)
print(f"{n:b}")    # "11111111" -- binary
print(f"{n:o}")    # "377"   -- octal
print(f"{n:x}")    # "ff"    -- hex lowercase
print(f"{n:X}")    # "FF"    -- hex uppercase
print(f"{n:#x}")   # "0xff"  -- hex with 0x prefix
print(f"{n:#b}")   # "0b11111111" -- binary with 0b prefix

# Triple quotes span multiple lines
poem = """Roses are red,
Violets are blue,
Python is great,
And so are you."""

print(poem)
print(len(poem.splitlines()))  # 4 lines

# Useful for SQL queries, HTML templates, etc.
query = """
    SELECT name, score
    FROM students
    WHERE score >= 90
    ORDER BY score DESC
    LIMIT 10
"""

# Multi-line in an expression (use backslash to avoid leading newline)
message = (
    "Dear Alice,\n"
    "Your order has shipped.\n"
    "Expected delivery: Monday."
)
# Adjacent string literals are automatically concatenated at compile time!

# In regular strings, backslash has special meaning (escape sequences)
print("C:\\Users\\Alice\\Documents")  # C:\Users\Alice\Documents
print("\t tab \n newline")            # actual tab and newline

# Raw strings (r"...") treat backslashes as literal characters
path = r"C:\Users\Alice\Documents"
print(path)           # C:\Users\Alice\Documents  (no need to double-escape)
print(len(path))      # 25  (backslashes are real chars)

# Critical use: regular expressions (backslashes are common in regex)
import re

# Without raw string: double every backslash
pattern1 = "\\d+\\.\\d+"           # matches digits.digits

# With raw string: much cleaner!
pattern2 = r"\d+\.\d+"             # same pattern, but readable

text = "Pi is approximately 3.14159"
match1 = re.findall(pattern1, text)
match2 = re.findall(pattern2, text)
print(match1)  # ['3.14159']
print(match2)  # ['3.14159']  (same result)

# Windows file paths
windows_path = r"C:\Program Files\Python\python.exe"
print(windows_path)  # no escaping needed!

# str is a sequence of Unicode characters (text)
text = "Hello"
print(type(text))   # <class 'str'>

# bytes is a sequence of raw bytes (0-255) -- for network/file I/O
data = b"Hello"
print(type(data))   # <class 'bytes'>
print(data[0])      # 72  (ASCII code for 'H')

# You CANNOT mix str and bytes
try:
    result = "Hello" + b" World"
except TypeError as e:
    print(f"Error: {e}")   # can only concatenate str (not "bytes") to str

# Converting between str and bytes: encode / decode
text = "Hello, World!"
encoded = text.encode("utf-8")     # str --> bytes
print(encoded)                     # b'Hello, World!'
print(type(encoded))               # <class 'bytes'>

decoded = encoded.decode("utf-8")  # bytes --> str
print(decoded)                     # "Hello, World!"
print(type(decoded))               # <class 'str'>

# ord() gives the code point of a character
print(ord('A'))          # 65
print(ord('a'))          # 97
print(ord('0'))          # 48
print(ord(' '))          # 32
print(ord('€'))          # 8364
print(ord('中'))          # 20013
print(ord('😀'))         # 128512

# chr() converts a code point back to a character
print(chr(65))           # 'A'
print(chr(97))           # 'a'
print(chr(8364))         # '€'
print(chr(128512))       # '😀'

# Strings ARE Unicode -- you can include any character
greeting = "Привет мир"         # Russian
print(greeting)
print(len(greeting))            # 10 (characters, not bytes)

# Unicode escapes in string literals
print("\u0041")          # 'A'   (4-hex-digit Unicode escape)
print("\U0001F600")      # '😀'  (8-hex-digit for codepoints > 0xFFFF)
print("\N{SNOWMAN}")     # '☃'   (Unicode name)

# UTF-8 encoding -- bytes per character
import sys

chars = ['A', 'é', '中', '😀']
for char in chars:
    encoded = char.encode('utf-8')
    print(f"{char!r:5} code point: {ord(char):7}  UTF-8 bytes: {len(encoded)}  hex: {encoded.hex()}")

# A     code point:      65  UTF-8 bytes: 1  hex: 41
# 'é'   code point:     233  UTF-8 bytes: 2  hex: c3a9
# '中'  code point:   20013  UTF-8 bytes: 3  hex: e4b8ad
# '😀'  code point:  128512  UTF-8 bytes: 4  hex: f09f9880

# This is why len(string) != len(bytes) for non-ASCII
text = "Hello, 中文!"
print(f"len(str):   {len(text)}")            # 10 characters
print(f"len(bytes): {len(text.encode())}")   # 14 bytes (中 and 文 are 3 bytes each)

# Reading files -- always specify encoding
with open("data.txt", "w", encoding="utf-8") as f:
    f.write("Hello, 中文! Привет!")

with open("data.txt", "r", encoding="utf-8") as f:
    content = f.read()

print(content)   # correct

# What happens without encoding on Windows (may default to cp1252)
# You get: UnicodeDecodeError or garbage characters
# ALWAYS specify encoding="utf-8" when reading/writing text files

# API responses: decode bytes to str
import json
json_bytes = b'{"name": "Alice", "score": 95}'
text = json_bytes.decode("utf-8")
data = json.loads(text)
print(data["name"])   # "Alice"

# THE WRONG WAY -- O(n^2) string concatenation
import time

words = ["word"] * 10000

# Method 1: += in a loop
start = time.perf_counter()
result = ""
for word in words:
    result += word + " "
t1 = time.perf_counter() - start

# Method 2: join (correct way)
start = time.perf_counter()
result = " ".join(words)
t2 = time.perf_counter() - start

print(f"Loop += : {t1*1000:.3f} ms")
print(f"join()  : {t2*1000:.3f} ms")
print(f"join is {t1/t2:.0f}x faster")
# join is commonly 10-100x faster for large inputs

# Building strings the CORRECT way
# Pattern 1: collect then join
parts = []
for i in range(1000):
    parts.append(f"item_{i}")
result = ", ".join(parts)

# Pattern 2: use a list comprehension then join (even cleaner)
result = ", ".join(f"item_{i}" for i in range(1000))

# Pattern 3: for simple cases, f-strings with io.StringIO
from io import StringIO
buf = StringIO()
for i in range(1000):
    buf.write(f"item_{i}\n")
result = buf.getvalue()

# Building HTML/CSV/SQL -- always use join or specialized libraries
headers = ["id", "name", "score"]
csv_header = ",".join(headers)
print(csv_header)   # "id,name,score"

rows = [[1, "Alice", 95], [2, "Bob", 78]]
csv_rows = [",".join(str(cell) for cell in row) for row in rows]
csv_output = "\n".join([csv_header] + csv_rows)
print(csv_output)

import re

text = "Alice scored 95 points on 2024-03-15. Bob scored 78 on 2024-03-16."

# re.findall() -- find all matches, return as list
numbers = re.findall(r'\d+', text)
print(numbers)        # ['95', '2024', '03', '15', '78', '2024', '03', '16']

# More specific: dates (YYYY-MM-DD format)
dates = re.findall(r'\d{4}-\d{2}-\d{2}', text)
print(dates)          # ['2024-03-15', '2024-03-16']

# Names followed by "scored N"
pattern = r'(\w+) scored (\d+)'
matches = re.findall(pattern, text)
print(matches)        # [('Alice', '95'), ('Bob', '78')]

# re.match() -- match at START of string
# re.search() -- search anywhere in string
# re.fullmatch() -- match ENTIRE string

import re

email_pattern = r'^[\w.+-]+@[\w-]+\.[a-zA-Z]{2,}$'

emails = ["alice@example.com", "bob@test.org", "not-an-email", "bad@.com"]
for email in emails:
    if re.fullmatch(email_pattern, email):
        print(f"VALID:   {email}")
    else:
        print(f"INVALID: {email}")

# ── 1. Clean text for NLP / LLM input ────────────────────────────────
def clean_text(text: str) -> str:
    """Normalize text for feeding to an LLM."""
    text = text.strip()                          # remove leading/trailing whitespace
    text = " ".join(text.split())                # collapse multiple spaces to one
    text = text.lower()                          # lowercase (for embeddings, not LLM input)
    # Remove common noise
    for char in ["  ", "\n\n", "\t"]:
        text = text.replace(char, " ")
    return text

# ── 2. Build LLM prompts with f-strings ──────────────────────────────
def build_prompt(user_question: str, context_chunks: list[str]) -> str:
    """Construct a RAG prompt from retrieved document chunks."""
    context = "\n\n".join(f"[{i+1}] {chunk}" for i, chunk in enumerate(context_chunks))
    return f"""You are a helpful assistant. Answer the question using ONLY the context below.
If the answer is not in the context, say "I don't know."

Context:
{context}

Question: {user_question}

Answer:"""

# ── 3. Parse structured data from LLM responses ──────────────────────
def extract_json_from_response(response: str) -> str:
    """Extract JSON from a response that might have extra text around it."""
    # LLMs often wrap JSON in markdown code blocks
    if "```json" in response:
        start = response.index("```json") + 7
        end = response.index("```", start)
        return response[start:end].strip()
    if "```" in response:
        start = response.index("```") + 3
        end = response.index("```", start)
        return response[start:end].strip()
    return response.strip()

# ── 4. Tokenize text manually (before using a real tokenizer) ─────────
def simple_tokenize(text: str) -> list[str]:
    """Split text into word tokens — how early NLP systems worked."""
    # Real tokenizers (BPE, WordPiece) are more sophisticated, but this is the concept
    return [word.strip(".,!?;:\"'()[]{}") for word in text.lower().split() if word]

text = "Machine learning, at its core, is pattern recognition!"
tokens = simple_tokenize(text)
# ['machine', 'learning', 'at', 'its', 'core', 'is', 'pattern', 'recognition']
print(f"Token count: {len(tokens)}")