Python os Module

A standard library module providing portable operating system interfaces for file system operations, environment variables, and process management.

Overview

The os module provides a unified interface for interacting with the operating system across different platforms (Unix, Windows, macOS). It wraps platform-specific functionality into portable Python functions, making it essential for file system navigation, process control, and environment management. For path manipulations, use os.path submodule or the modern pathlib alternative.

⤢ zoom

Path Operations

The os.path submodule provides functions for manipulating file paths in a platform-independent manner.

Key Concepts

• Path joining: Combine path components using the correct separator for the OS
• Path splitting: Extract directory, filename, and extension components
• Path validation: Check existence and type of paths
• Normalisation: Resolve relative paths and remove redundant separators

Common Patterns

import os

# Join path components (handles separators automatically)
config_path = os.path.join('/etc', 'app', 'config.yaml')
# Result: '/etc/app/config.yaml' (Unix) or '\\etc\\app\\config.yaml' (Windows)

# Get directory and filename from path
directory = os.path.dirname('/home/user/data/report.csv')  # '/home/user/data'
filename = os.path.basename('/home/user/data/report.csv')  # 'report.csv'

# Split path into directory and filename
dir_part, file_part = os.path.split('/home/user/data/report.csv')
# dir_part: '/home/user/data', file_part: 'report.csv'

# Split filename and extension
name, ext = os.path.splitext('report.csv')  # ('report', '.csv')

# Check path existence and type
os.path.exists('/tmp/myfile.txt')   # True if path exists
os.path.isfile('/tmp/myfile.txt')   # True if it's a file
os.path.isdir('/tmp/mydir')         # True if it's a directory
os.path.islink('/tmp/mylink')       # True if it's a symbolic link

# Get absolute and real paths
os.path.abspath('relative/path')    # Convert to absolute path
os.path.realpath('/tmp/link')       # Resolve symbolic links

# Normalise path (remove redundant separators and resolve ..)
os.path.normpath('/home/user/../user/./docs')  # '/home/user/docs'

Examples

Building cross-platform paths:

import os

# Get user's home directory
home = os.path.expanduser('~')

# Build path to config file
config_dir = os.path.join(home, '.config', 'myapp')
config_file = os.path.join(config_dir, 'settings.json')

# Check if config exists, create if not
if not os.path.exists(config_dir):
    os.makedirs(config_dir)

# Get file size if file exists
if os.path.isfile(config_file):
    size = os.path.getsize(config_file)
    print(f"Config size: {size} bytes")

Processing files with specific extensions:

import os

def find_python_files(directory):
    """Find all Python files in a directory."""
    python_files = []

    for item in os.listdir(directory):
        full_path = os.path.join(directory, item)

        if os.path.isfile(full_path):
            name, ext = os.path.splitext(item)
            if ext.lower() == '.py':
                python_files.append(full_path)

    return python_files

# Usage
scripts = find_python_files('/home/user/projects')

Relative path calculations:

import os

# Get relative path between two locations
source = '/home/user/projects/app/src/main.py'
target = '/home/user/projects/app/data/config.json'

relative = os.path.relpath(target, os.path.dirname(source))
# Result: '../data/config.json'

# Check if path is absolute
os.path.isabs('/usr/bin/python')    # True
os.path.isabs('relative/path')       # False

# Get common prefix of paths
paths = ['/home/user/docs', '/home/user/data', '/home/user/projects']
common = os.path.commonpath(paths)  # '/home/user'

Directory Operations

Functions for creating, listing, traversing, and removing directories.

Key Concepts

• Directory listing: Get contents of a directory (files and subdirectories)
• Directory creation: Create single or nested directories
• Directory traversal: Recursively walk through directory trees
• Working directory: Get and change the current working directory

Common Patterns

import os

# List directory contents
items = os.listdir('/home/user/projects')  # Returns list of names (not full paths)

# Efficient directory scanning with metadata
with os.scandir('/home/user/projects') as entries:
    for entry in entries:
        print(f"{entry.name}: {'dir' if entry.is_dir() else 'file'}")

# Create single directory
os.mkdir('/tmp/newdir')  # Fails if parent doesn't exist or dir exists

# Create nested directories
os.makedirs('/tmp/path/to/newdir', exist_ok=True)  # exist_ok prevents error

# Remove empty directory
os.rmdir('/tmp/emptydir')

# Remove nested empty directories
os.removedirs('/tmp/path/to/emptydir')  # Removes from leaf up while empty

# Get current working directory
cwd = os.getcwd()

# Change working directory
os.chdir('/home/user/projects')

Examples

Recursively walk directory tree:

import os

def get_directory_size(path):
    """Calculate total size of all files in directory tree."""
    total_size = 0

    for dirpath, dirnames, filenames in os.walk(path):
        for filename in filenames:
            filepath = os.path.join(dirpath, filename)
            # Skip symbolic links
            if not os.path.islink(filepath):
                total_size += os.path.getsize(filepath)

    return total_size

# Usage
size = get_directory_size('/home/user/documents')
print(f"Total size: {size / 1024 / 1024:.2f} MB")

Clean up empty directories:

import os

def remove_empty_dirs(root_path):
    """Remove all empty directories under root_path."""
    removed = []

    # Walk bottom-up to remove nested empty dirs first
    for dirpath, dirnames, filenames in os.walk(root_path, topdown=False):
        if not dirnames and not filenames:
            try:
                os.rmdir(dirpath)
                removed.append(dirpath)
            except OSError as e:
                print(f"Could not remove {dirpath}: {e}")

    return removed

Backup directory structure:

import os

def mirror_directory_structure(source, destination):
    """Create empty directory structure matching source."""
    for dirpath, dirnames, filenames in os.walk(source):
        # Calculate relative path
        rel_path = os.path.relpath(dirpath, source)
        dest_path = os.path.join(destination, rel_path)

        # Create directory if it doesn't exist
        os.makedirs(dest_path, exist_ok=True)
        print(f"Created: {dest_path}")

# Usage
mirror_directory_structure('/home/user/project', '/backup/project')

Using scandir for better performance:

import os

def list_files_with_info(directory):
    """List files with size and modification time."""
    files_info = []

    with os.scandir(directory) as entries:
        for entry in entries:
            if entry.is_file():
                stat_info = entry.stat()
                files_info.append({
                    'name': entry.name,
                    'path': entry.path,
                    'size': stat_info.st_size,
                    'modified': stat_info.st_mtime
                })

    return sorted(files_info, key=lambda x: x['modified'], reverse=True)

Environment Variables

Functions for reading and modifying environment variables.

Key Concepts

• Environment access: Read environment variables with optional defaults
• Environment dictionary: Direct access to all environment variables
• Modification: Set or remove environment variables (affects child processes)
• Path handling: Work with PATH and similar list-based variables

Common Patterns

import os

# Get environment variable with default
home = os.getenv('HOME', '/tmp')
debug = os.getenv('DEBUG', 'false')

# Get required environment variable (raises KeyError if missing)
api_key = os.environ['API_KEY']

# Check if variable exists
if 'DATABASE_URL' in os.environ:
    db_url = os.environ['DATABASE_URL']

# Set environment variable (inherited by child processes)
os.environ['APP_ENV'] = 'production'

# Remove environment variable
del os.environ['TEMP_VAR']
# Or use pop with default
os.environ.pop('OPTIONAL_VAR', None)

# Get all environment variables
all_vars = dict(os.environ)

# Expand environment variables in strings
path = os.path.expandvars('$HOME/.config/app')  # Expands $HOME

Examples

Configuration from environment:

import os

class Config:
    """Application configuration from environment variables."""

    def __init__(self):
        self.debug = os.getenv('DEBUG', 'false').lower() == 'true'
        self.host = os.getenv('HOST', '0.0.0.0')
        self.port = int(os.getenv('PORT', '8000'))
        self.database_url = os.environ.get('DATABASE_URL')
        self.secret_key = os.environ.get('SECRET_KEY')

        # Validate required variables
        if not self.secret_key:
            raise ValueError("SECRET_KEY environment variable is required")

    @property
    def is_production(self):
        return os.getenv('APP_ENV') == 'production'

# Usage
config = Config()
print(f"Running on {config.host}:{config.port}")

Working with PATH variable:

import os

def add_to_path(directory):
    """Add directory to system PATH."""
    current_path = os.environ.get('PATH', '')

    # Check if already in PATH
    path_dirs = current_path.split(os.pathsep)
    if directory not in path_dirs:
        new_path = directory + os.pathsep + current_path
        os.environ['PATH'] = new_path
        return True
    return False

def find_in_path(executable):
    """Find executable in PATH directories."""
    path_dirs = os.environ.get('PATH', '').split(os.pathsep)

    for directory in path_dirs:
        full_path = os.path.join(directory, executable)
        if os.path.isfile(full_path) and os.access(full_path, os.X_OK):
            return full_path

    return None

# Usage
python_path = find_in_path('python3')

Environment variable substitution:

import os

def load_config_with_env(config_dict):
    """Replace ${VAR} patterns with environment variable values."""
    result = {}

    for key, value in config_dict.items():
        if isinstance(value, str) and '${' in value:
            # Find and replace all ${VAR} patterns
            import re
            def replace_var(match):
                var_name = match.group(1)
                return os.getenv(var_name, match.group(0))

            result[key] = re.sub(r'\$\{(\w+)\}', replace_var, value)
        else:
            result[key] = value

    return result

# Usage
config = {
    'database': '${DB_HOST}:${DB_PORT}/${DB_NAME}',
    'timeout': 30
}
resolved = load_config_with_env(config)

File Operations

Functions for manipulating files (delete, rename, get metadata).

Key Concepts

• File removal: Delete files from the filesystem
• File renaming/moving: Rename or move files within the same filesystem
• File metadata: Get detailed information about files (size, timestamps, permissions)
• Permissions: Change file permissions and ownership

Common Patterns

import os
import stat

# Remove a file
os.remove('/tmp/oldfile.txt')
os.unlink('/tmp/oldfile.txt')  # Alias for remove

# Rename or move file
os.rename('/tmp/old.txt', '/tmp/new.txt')
os.replace('/tmp/source.txt', '/tmp/dest.txt')  # Atomic, overwrites dest

# Get file metadata
file_stat = os.stat('/tmp/myfile.txt')
print(f"Size: {file_stat.st_size} bytes")
print(f"Modified: {file_stat.st_mtime}")  # Unix timestamp
print(f"Metadata changed: {file_stat.st_ctime}")  # inode change time on Linux/macOS, NOT creation time
print(f"Permissions: {oct(file_stat.st_mode)}")
print(f"Owner UID: {file_stat.st_uid}")

# Check file permissions
os.access('/tmp/script.sh', os.R_OK)  # Readable
os.access('/tmp/script.sh', os.W_OK)  # Writable
os.access('/tmp/script.sh', os.X_OK)  # Executable
os.access('/tmp/script.sh', os.F_OK)  # Exists

# Change file permissions
os.chmod('/tmp/script.sh', 0o755)  # rwxr-xr-x
os.chmod('/tmp/private.txt', stat.S_IRUSR | stat.S_IWUSR)  # rw-------

# Change file ownership (Unix only, requires appropriate privileges)
os.chown('/tmp/myfile.txt', uid=1000, gid=1000)

Examples

Safe file operations with error handling:

import os

def safe_remove(filepath):
    """Remove file if it exists, return success status."""
    try:
        if os.path.isfile(filepath):
            os.remove(filepath)
            return True
        return False
    except PermissionError:
        print(f"Permission denied: {filepath}")
        return False
    except OSError as e:
        print(f"Error removing {filepath}: {e}")
        return False

def safe_rename(source, destination, overwrite=False):
    """Safely rename/move file with optional overwrite."""
    if not os.path.exists(source):
        raise FileNotFoundError(f"Source not found: {source}")

    if os.path.exists(destination):
        if overwrite:
            os.remove(destination)
        else:
            raise FileExistsError(f"Destination exists: {destination}")

    # Create destination directory if needed
    dest_dir = os.path.dirname(destination)
    if dest_dir and not os.path.exists(dest_dir):
        os.makedirs(dest_dir)

    os.rename(source, destination)

File metadata utilities:

import os
from datetime import datetime

def get_file_info(filepath):
    """Get comprehensive file information."""
    if not os.path.exists(filepath):
        raise FileNotFoundError(filepath)

    stat_info = os.stat(filepath)

    return {
        'path': os.path.abspath(filepath),
        'name': os.path.basename(filepath),
        'size_bytes': stat_info.st_size,
        'size_human': format_size(stat_info.st_size),
        'metadata_changed': datetime.fromtimestamp(stat_info.st_ctime),  # inode change time, not creation
        'modified': datetime.fromtimestamp(stat_info.st_mtime),
        'accessed': datetime.fromtimestamp(stat_info.st_atime),
        'permissions': oct(stat_info.st_mode)[-3:],
        'is_symlink': os.path.islink(filepath)
    }

def format_size(size_bytes):
    """Format bytes as human-readable string."""
    for unit in ['B', 'KB', 'MB', 'GB', 'TB']:
        if size_bytes < 1024:
            return f"{size_bytes:.2f} {unit}"
        size_bytes /= 1024
    return f"{size_bytes:.2f} PB"

# Usage
info = get_file_info('/var/log/syslog')
print(f"File: {info['name']}, Size: {info['size_human']}")

Batch file operations:

import os

def cleanup_old_files(directory, days=30):
    """Remove files older than specified days."""
    import time

    cutoff_time = time.time() - (days * 86400)
    removed = []

    for filename in os.listdir(directory):
        filepath = os.path.join(directory, filename)

        if os.path.isfile(filepath):
            if os.stat(filepath).st_mtime < cutoff_time:
                os.remove(filepath)
                removed.append(filepath)

    return removed

def organise_by_extension(source_dir, dest_dir):
    """Move files into subdirectories by extension."""
    for filename in os.listdir(source_dir):
        source_path = os.path.join(source_dir, filename)

        if os.path.isfile(source_path):
            _, ext = os.path.splitext(filename)
            ext = ext.lower().strip('.') or 'no_extension'

            ext_dir = os.path.join(dest_dir, ext)
            os.makedirs(ext_dir, exist_ok=True)

            dest_path = os.path.join(ext_dir, filename)
            os.rename(source_path, dest_path)

Process Management

Functions for executing commands, managing processes, and interacting with the system.

Key Concepts

• Command execution: Run shell commands and capture output
• Process replacement: Replace current process with new program (exec family)
• Process forking: Create child processes (Unix)
• Process information: Get process IDs and send signals

Common Patterns

import os
import subprocess

# Execute shell command (prefer subprocess for new code)
exit_code = os.system('ls -la /tmp')  # Returns exit status

# Get current process ID
pid = os.getpid()
ppid = os.getppid()  # Parent process ID

# Get process group and session
pgid = os.getpgid(0)  # 0 means current process
sid = os.getsid(0)

# Send signal to process
os.kill(pid, 0)  # Check if process exists (signal 0)
# os.kill(pid, signal.SIGTERM)  # Terminate process
# os.kill(pid, signal.SIGKILL)  # Force kill

# Execute and replace current process (Unix)
# os.execvp('python', ['python', 'script.py', 'arg1'])

# Fork process (Unix only)
# pid = os.fork()
# if pid == 0:
#     # Child process
#     pass
# else:
#     # Parent process
#     pass

Examples

Basic command execution:

import os

# Simple command execution
result = os.system('echo "Hello World"')
if result == 0:
    print("Command succeeded")
else:
    print(f"Command failed with code {result}")

# Recommended: Use subprocess for better control
import subprocess

# Run command and capture output
result = subprocess.run(
    ['ls', '-la', '/tmp'],
    capture_output=True,
    text=True
)
print(f"stdout: {result.stdout}")
print(f"stderr: {result.stderr}")
print(f"return code: {result.returncode}")

# Run with shell expansion
result = subprocess.run(
    'echo $HOME/*.py',
    shell=True,
    capture_output=True,
    text=True
)

Process forking (Unix):

import os
import sys

def run_in_background(func, *args):
    """Run function in background process."""
    pid = os.fork()

    if pid == 0:
        # Child process
        try:
            func(*args)
        finally:
            os._exit(0)  # Exit child without cleanup
    else:
        # Parent process
        return pid  # Return child PID

def worker(task_id, data):
    """Example worker function."""
    print(f"Worker {task_id} processing: {data}")
    import time
    time.sleep(2)
    print(f"Worker {task_id} done")

# Usage (Unix only)
# child_pid = run_in_background(worker, 1, "task data")
# print(f"Started worker with PID {child_pid}")
# os.waitpid(child_pid, 0)  # Wait for child to finish

Daemon process creation (Unix):

import os
import sys

def daemonise():
    """Convert current process to daemon."""
    # First fork
    pid = os.fork()
    if pid > 0:
        sys.exit(0)  # Parent exits

    # Create new session
    os.setsid()

    # Second fork (prevent acquiring terminal)
    pid = os.fork()
    if pid > 0:
        sys.exit(0)

    # Change working directory
    os.chdir('/')

    # Close standard file descriptors
    os.close(0)
    os.close(1)
    os.close(2)

    # Redirect to /dev/null
    fd = os.open('/dev/null', os.O_RDWR)
    os.dup2(fd, 0)
    os.dup2(fd, 1)
    os.dup2(fd, 2)

    return os.getpid()

Process execution with exec:

import os

def exec_python_script(script_path, *args):
    """Replace current process with Python script."""
    # execvp searches PATH for executable
    os.execvp('python3', ['python3', script_path] + list(args))
    # Code after exec never runs (process replaced)

def exec_with_environment(command, args, env_vars):
    """Execute command with custom environment."""
    # Copy current environment and add new variables
    env = os.environ.copy()
    env.update(env_vars)

    # execvpe allows specifying environment
    os.execvpe(command, [command] + args, env)

# Usage (replaces current process)
# exec_with_environment('node', ['app.js'], {'NODE_ENV': 'production'})

Process monitoring:

import os
import signal

def is_process_running(pid):
    """Check if process with given PID is running."""
    try:
        os.kill(pid, 0)
        return True
    except ProcessLookupError:
        return False
    except PermissionError:
        # Process exists but we don't have permission
        return True

def terminate_gracefully(pid, timeout=5):
    """Try graceful termination, then force kill."""
    import time

    if not is_process_running(pid):
        return True

    # Send SIGTERM
    os.kill(pid, signal.SIGTERM)

    # Wait for process to terminate
    start = time.time()
    while time.time() - start < timeout:
        if not is_process_running(pid):
            return True
        time.sleep(0.1)

    # Force kill
    os.kill(pid, signal.SIGKILL)
    return False

Quick Reference

Task	Code
Join paths	os.path.join(dir, subdir, file)
Get dirname	os.path.dirname(path)
Get basename	os.path.basename(path)
Check exists	os.path.exists(path)
Is file	os.path.isfile(path)
Is directory	os.path.isdir(path)
List directory	os.listdir(path)
Create directory	os.makedirs(path, exist_ok=True)
Walk tree	os.walk(path)
Get cwd	os.getcwd()
Change cwd	os.chdir(path)
Get env var	os.getenv('VAR', default)
Set env var	os.environ['VAR'] = value
Remove file	os.remove(path)
Rename file	os.rename(old, new)
File stats	os.stat(path)
File size	os.path.getsize(path)
Run command	os.system(command)
Get PID	os.getpid()

Common Imports

import os
import os.path
from os import environ, getenv
from os.path import join, dirname, basename, exists

# For modern path handling (alternative)
from pathlib import Path

Platform-Specific Constants

import os

os.name          # 'posix', 'nt', 'java'
os.sep           # Path separator ('/' or '\\')
os.pathsep       # PATH separator (':' or ';')
os.linesep       # Line ending ('\n', '\r\n', '\r')
os.devnull       # Null device ('/dev/null' or 'nul')
os.curdir        # Current directory ('.')
os.pardir        # Parent directory ('..')

Common Issues and Solutions

Issue	Solution
FileNotFoundError on path operations	Use os.path.exists() before operations; check working directory with os.getcwd()
PermissionError when removing/writing	Check permissions with os.access(); use appropriate user/sudo
Path separator issues across platforms	Always use os.path.join() instead of string concatenation
OSError: Directory not empty	Use shutil.rmtree() to remove non-empty directories
Environment variable not found	Use os.getenv('VAR', default) instead of os.environ['VAR']
os.rename() fails across filesystems	Use shutil.move() for cross-filesystem moves
Symbolic link issues	Use os.path.realpath() to resolve links; check with os.path.islink()
os.walk() permission errors	Wrap in try-except or use onerror parameter
Process fork not available	Fork is Unix-only; use subprocess or multiprocessing on Windows
Relative paths confusion	Convert to absolute with os.path.abspath() before operations

Debugging Tips

import os

# Print environment for debugging
def debug_env():
    print(f"CWD: {os.getcwd()}")
    print(f"PATH: {os.environ.get('PATH', '')}")
    print(f"HOME: {os.path.expanduser('~')}")
    print(f"Platform: {os.name}")

# Check file details
def debug_file(path):
    print(f"Path: {path}")
    print(f"Exists: {os.path.exists(path)}")
    print(f"Absolute: {os.path.abspath(path)}")
    if os.path.exists(path):
        stat = os.stat(path)
        print(f"Size: {stat.st_size}")
        print(f"Mode: {oct(stat.st_mode)}")

# Trace directory contents
def debug_dir(path, max_items=20):
    print(f"Contents of {path}:")
    try:
        items = os.listdir(path)[:max_items]
        for item in items:
            full = os.path.join(path, item)
            type_str = 'd' if os.path.isdir(full) else 'f'
            print(f"  [{type_str}] {item}")
    except PermissionError:
        print("  Permission denied")

Modern Alternative: pathlib

from pathlib import Path

# pathlib equivalents for common operations
p = Path('/home/user/file.txt')

p.exists()              # os.path.exists
p.is_file()             # os.path.isfile
p.is_dir()              # os.path.isdir
p.parent                # os.path.dirname
p.name                  # os.path.basename
p.stem                  # filename without extension
p.suffix                # extension
p.stat().st_size        # os.path.getsize

# Path manipulation
p / 'subdir' / 'file'   # os.path.join
p.resolve()             # os.path.abspath
p.read_text()           # open + read
p.write_text('data')    # open + write

# Directory operations
Path('.').iterdir()     # os.listdir
Path('.').glob('*.py')  # filtered listing
p.mkdir(parents=True)   # os.makedirs

Related Topics

The following topics complement Python os module usage:

1. Python - pathlib: Modern object-oriented path handling (recommended for new code)
2. Python - shutil: High-level file operations (copy, move, archive)
3. Python - subprocess: Advanced process execution with output capture
4. Python - tempfile: Secure temporary file and directory creation
5. Python - glob: Pattern matching for file paths
6. Python - stat: File permission constants and mode parsing