Python-Magic

File type identification library using libmagic for accurate MIME type detection and metadata extraction.

Overview

python-magic is a Python wrapper around the libmagic file identification library. It identifies file types by examining file contents rather than relying on file extensions, making it essential for secure file validation, content inspection, and upload handling in web applications.

⤢ zoom

Installation

Package Installation

# Install python-magic
pip install python-magic

# On Ubuntu/Debian - install libmagic
sudo apt-get install libmagic1

# On macOS
brew install libmagic

# On Windows - use python-magic-bin instead
pip install python-magic-bin

Verify Installation

import magic

# Check installation works
mime = magic.Magic(mime=True)
print(mime.from_file("test.txt"))

File Type Detection

Core functionality for identifying file types from files and buffers.

Basic File Detection

import magic

# Get human-readable description
description = magic.from_file("document.pdf")
# Returns: 'PDF document, version 1.4'

# Get MIME type
mime_type = magic.from_file("document.pdf", mime=True)
# Returns: 'application/pdf'

# Using Magic class for reusable instances
m = magic.Magic()
print(m.from_file("image.png"))
# Returns: 'PNG image data, 800 x 600, 8-bit/color RGBA, non-interlaced'

Buffer Detection

import magic

# Detect from bytes
with open("document.pdf", "rb") as f:
    data = f.read()

# Get description from buffer
description = magic.from_buffer(data)

# Get MIME type from buffer
mime_type = magic.from_buffer(data, mime=True)

# Using Magic class
m = magic.Magic(mime=True)
mime_type = m.from_buffer(data)

Detection Workflow

⤢ zoom

MIME Type Detection

Detecting and working with MIME types for content negotiation and validation.

Basic MIME Detection

import magic

# Create MIME-specific detector
mime = magic.Magic(mime=True)

# Common file types
print(mime.from_file("photo.jpg"))      # image/jpeg
print(mime.from_file("data.json"))      # application/json
print(mime.from_file("script.py"))      # text/x-script.python
print(mime.from_file("archive.tar.gz")) # application/gzip

MIME with Encoding

import magic

# Include encoding information
mime_encoding = magic.Magic(mime=True, mime_encoding=True)

result = mime_encoding.from_file("document.txt")
# Returns: 'text/plain; charset=utf-8'

# Parse MIME and encoding
mime_type, _, encoding = result.partition("; charset=")
print(f"Type: {mime_type}, Encoding: {encoding}")

MIME Type Categories

Category	Common Types	Example Files
Images	image/jpeg, image/png, image/gif, image/webp	.jpg, .png, .gif
Documents	application/pdf, application/msword	.pdf, .doc
Text	text/plain, text/html, text/csv	.txt, .html, .csv
Archives	application/zip, application/gzip	.zip, .tar.gz
Audio	audio/mpeg, audio/wav	.mp3, .wav
Video	video/mp4, video/webm	.mp4, .webm

Metadata Extraction

Extracting detailed information about file contents and structure.

Detailed File Information

import magic

# Get comprehensive description
m = magic.Magic()

# Image metadata
print(m.from_file("photo.jpg"))
# 'JPEG image data, JFIF standard 1.01, resolution (DPI),
#  density 72x72, segment length 16, baseline, precision 8,
#  1920x1080, components 3'

# Document metadata
print(m.from_file("document.pdf"))
# 'PDF document, version 1.7 (password protected)'

# Archive contents
print(m.from_file("archive.zip"))
# 'Zip archive data, at least v2.0 to extract'

Extracting Specific Metadata

import magic
import re

def get_image_dimensions(filepath):
    """Extract image dimensions from magic output."""
    m = magic.Magic()
    info = m.from_file(filepath)

    # Match common dimension patterns
    patterns = [
        r'(\d+)\s*x\s*(\d+)',           # WxH format
        r'(\d+)\s*×\s*(\d+)',           # W×H format
    ]

    for pattern in patterns:
        match = re.search(pattern, info)
        if match:
            return int(match.group(1)), int(match.group(2))

    return None

# Usage
width, height = get_image_dimensions("photo.png")
print(f"Dimensions: {width}x{height}")

Common Metadata Fields

import magic

def analyse_file(filepath):
    """Extract common metadata from any file."""
    m = magic.Magic()
    mime = magic.Magic(mime=True)
    encoding = magic.Magic(mime_encoding=True)

    return {
        "description": m.from_file(filepath),
        "mime_type": mime.from_file(filepath),
        "encoding": encoding.from_file(filepath),
    }

# Usage
info = analyse_file("document.pdf")
for key, value in info.items():
    print(f"{key}: {value}")

Common Use Cases

File Upload Validation

import magic
from pathlib import Path

class FileValidator:
    """Validate uploaded files by content type."""

    ALLOWED_TYPES = {
        "image/jpeg",
        "image/png",
        "image/gif",
        "image/webp",
        "application/pdf",
    }

    MAX_SIZE = 10 * 1024 * 1024  # 10MB

    def __init__(self):
        self.mime = magic.Magic(mime=True)

    def validate(self, file_data: bytes, filename: str) -> dict:
        """Validate file and return result."""
        errors = []

        # Check size
        if len(file_data) > self.MAX_SIZE:
            errors.append(f"File exceeds maximum size of {self.MAX_SIZE} bytes")

        # Check MIME type from content
        mime_type = self.mime.from_buffer(file_data)
        if mime_type not in self.ALLOWED_TYPES:
            errors.append(f"File type '{mime_type}' not allowed")

        # Verify extension matches content
        ext = Path(filename).suffix.lower()
        expected_ext = self._get_expected_extension(mime_type)
        if ext not in expected_ext:
            errors.append(f"Extension '{ext}' doesn't match content type '{mime_type}'")

        return {
            "valid": len(errors) == 0,
            "mime_type": mime_type,
            "errors": errors,
        }

    def _get_expected_extension(self, mime_type: str) -> set:
        """Map MIME types to expected extensions."""
        mapping = {
            "image/jpeg": {".jpg", ".jpeg"},
            "image/png": {".png"},
            "image/gif": {".gif"},
            "image/webp": {".webp"},
            "application/pdf": {".pdf"},
        }
        return mapping.get(mime_type, set())

# Usage
validator = FileValidator()
with open("upload.jpg", "rb") as f:
    result = validator.validate(f.read(), "upload.jpg")

if result["valid"]:
    print(f"Valid file: {result['mime_type']}")
else:
    print(f"Errors: {result['errors']}")

Content Inspection Pipeline

⤢ zoom

Batch File Processing

import magic
from pathlib import Path
from concurrent.futures import ThreadPoolExecutor

def process_directory(directory: str, allowed_types: set = None):
    """Process all files in directory and categorise by type."""
    mime = magic.Magic(mime=True)
    results = {"files": [], "by_type": {}}

    for filepath in Path(directory).rglob("*"):
        if filepath.is_file():
            mime_type = mime.from_file(str(filepath))

            # Filter if types specified
            if allowed_types and mime_type not in allowed_types:
                continue

            file_info = {
                "path": str(filepath),
                "mime_type": mime_type,
                "size": filepath.stat().st_size,
            }

            results["files"].append(file_info)

            # Group by type
            if mime_type not in results["by_type"]:
                results["by_type"][mime_type] = []
            results["by_type"][mime_type].append(str(filepath))

    return results

# Usage
results = process_directory("/uploads", {"image/jpeg", "image/png"})
for mime_type, files in results["by_type"].items():
    print(f"{mime_type}: {len(files)} files")

Custom Magic Databases

Using custom magic files for specialised file type detection.

Creating Custom Magic Files

# custom.magic file format:
# offset  type    test    message

# Example custom.magic file content:
"""
0       string  MYAPP   My Application Data File
>6      byte    x       version %d
"""

Using Custom Database

import magic

# Load custom magic database
custom_magic = magic.Magic(magic_file="/path/to/custom.magic")

# Combine with system database
combined = magic.Magic(magic_file="/path/to/custom.magic:/usr/share/misc/magic")

# Check file with custom rules
result = custom_magic.from_file("myapp.dat")

Compiling Magic Files

# Compile magic file for faster loading
file -C -m custom.magic

# This creates custom.magic.mgc

Custom Database Example

import magic
import tempfile
import os

def create_custom_detector():
    """Create detector for custom file format."""

    # Define custom magic rules
    custom_rules = """
# Custom application formats
0   string  \\x89PNG    PNG Image (custom)
0   string  %PDF       PDF Document (custom)
0   string  RIFF       RIFF container
>8  string  WEBP       \\b, WebP image
"""

    # Write to temporary file
    with tempfile.NamedTemporaryFile(mode='w', suffix='.magic', delete=False) as f:
        f.write(custom_rules)
        magic_file = f.name

    try:
        # Create magic instance with custom file
        m = magic.Magic(magic_file=magic_file)
        return m
    finally:
        # Cleanup
        os.unlink(magic_file)

Integration with Web Frameworks

Flask Integration

from flask import Flask, request, jsonify
import magic

app = Flask(__name__)

# Create reusable Magic instance
mime_detector = magic.Magic(mime=True)

ALLOWED_MIMES = {
    "image/jpeg",
    "image/png",
    "application/pdf",
}
MAX_SIZE = 5 * 1024 * 1024  # 5MB

@app.route("/upload", methods=["POST"])
def upload_file():
    if "file" not in request.files:
        return jsonify({"error": "No file provided"}), 400

    file = request.files["file"]

    if file.filename == "":
        return jsonify({"error": "No file selected"}), 400

    # Read file content
    content = file.read()

    # Check size
    if len(content) > MAX_SIZE:
        return jsonify({"error": "File too large"}), 413

    # Detect actual MIME type
    mime_type = mime_detector.from_buffer(content)

    # Validate type
    if mime_type not in ALLOWED_MIMES:
        return jsonify({
            "error": f"File type '{mime_type}' not allowed",
            "allowed": list(ALLOWED_MIMES),
        }), 415

    # Process file...
    file.seek(0)  # Reset file pointer

    return jsonify({
        "message": "File uploaded successfully",
        "mime_type": mime_type,
        "size": len(content),
    })

@app.route("/detect", methods=["POST"])
def detect_type():
    """Detect file type without saving."""
    if "file" not in request.files:
        return jsonify({"error": "No file provided"}), 400

    content = request.files["file"].read()

    m = magic.Magic()
    mime = magic.Magic(mime=True)

    return jsonify({
        "description": m.from_buffer(content),
        "mime_type": mime.from_buffer(content),
    })

FastAPI Integration

from fastapi import FastAPI, UploadFile, HTTPException, Depends
from fastapi.responses import JSONResponse
import magic
from typing import Set

app = FastAPI()

class FileTypeValidator:
    """Dependency for file type validation."""

    def __init__(self, allowed_types: Set[str], max_size: int):
        self.allowed_types = allowed_types
        self.max_size = max_size
        self.mime = magic.Magic(mime=True)

    async def __call__(self, file: UploadFile) -> UploadFile:
        # Read content
        content = await file.read()

        # Check size
        if len(content) > self.max_size:
            raise HTTPException(
                status_code=413,
                detail=f"File size {len(content)} exceeds limit {self.max_size}"
            )

        # Check type
        mime_type = self.mime.from_buffer(content)
        if mime_type not in self.allowed_types:
            raise HTTPException(
                status_code=415,
                detail=f"File type '{mime_type}' not allowed"
            )

        # Reset file position
        await file.seek(0)

        # Attach metadata
        file.mime_type = mime_type
        file.content_length = len(content)

        return file

# Create validators for different endpoints
image_validator = FileTypeValidator(
    allowed_types={"image/jpeg", "image/png", "image/webp"},
    max_size=10 * 1024 * 1024
)

document_validator = FileTypeValidator(
    allowed_types={"application/pdf", "application/msword"},
    max_size=50 * 1024 * 1024
)

@app.post("/upload/image")
async def upload_image(file: UploadFile = Depends(image_validator)):
    return {
        "filename": file.filename,
        "mime_type": file.mime_type,
        "size": file.content_length,
    }

@app.post("/upload/document")
async def upload_document(file: UploadFile = Depends(document_validator)):
    return {
        "filename": file.filename,
        "mime_type": file.mime_type,
        "size": file.content_length,
    }

@app.post("/detect")
async def detect_file_type(file: UploadFile):
    """Detect file type without validation."""
    content = await file.read()

    m = magic.Magic()
    mime = magic.Magic(mime=True)
    encoding = magic.Magic(mime_encoding=True)

    return {
        "filename": file.filename,
        "description": m.from_buffer(content),
        "mime_type": mime.from_buffer(content),
        "encoding": encoding.from_buffer(content),
    }

Request Flow Diagram

⤢ zoom

Security Considerations

Defence Against File Type Spoofing

import magic
from pathlib import Path

class SecureFileValidator:
    """Security-focused file validator."""

    # Dangerous MIME types to always reject
    DANGEROUS_TYPES = {
        "application/x-executable",
        "application/x-msdos-program",
        "application/x-msdownload",
        "application/x-sh",
        "application/x-shellscript",
        "text/x-sh",
        "text/x-python",
        "text/x-perl",
        "application/x-php",
    }

    # Polyglot detection patterns
    POLYGLOT_SIGNATURES = [
        (b"GIF89a", b"<script"),  # GIF with JavaScript
        (b"\x89PNG", b"<?php"),    # PNG with PHP
        (b"%PDF", b"<script"),     # PDF with JavaScript
    ]

    def __init__(self):
        self.mime = magic.Magic(mime=True)
        self.detail = magic.Magic()

    def validate_secure(self, content: bytes, filename: str) -> dict:
        """Perform security-focused validation."""
        issues = []

        # Get MIME type
        mime_type = self.mime.from_buffer(content)

        # Check for dangerous types
        if mime_type in self.DANGEROUS_TYPES:
            issues.append(f"Dangerous file type: {mime_type}")

        # Check for extension mismatch (potential spoofing)
        ext = Path(filename).suffix.lower()
        if not self._extension_matches_type(ext, mime_type):
            issues.append(f"Extension '{ext}' doesn't match detected type '{mime_type}'")

        # Check for polyglot files
        if self._is_polyglot(content):
            issues.append("Potential polyglot file detected")

        # Check for null bytes in text files
        if mime_type.startswith("text/") and b"\x00" in content:
            issues.append("Null bytes in text file")

        # Check for double extensions
        if self._has_double_extension(filename):
            issues.append("Double extension detected")

        return {
            "secure": len(issues) == 0,
            "mime_type": mime_type,
            "issues": issues,
        }

    def _extension_matches_type(self, ext: str, mime_type: str) -> bool:
        """Check if extension matches MIME type."""
        type_extensions = {
            "image/jpeg": {".jpg", ".jpeg"},
            "image/png": {".png"},
            "image/gif": {".gif"},
            "application/pdf": {".pdf"},
            "text/plain": {".txt", ".text"},
            "text/html": {".html", ".htm"},
        }

        expected = type_extensions.get(mime_type)
        if expected is None:
            return True  # Unknown type, allow

        return ext in expected

    def _is_polyglot(self, content: bytes) -> bool:
        """Check for polyglot file signatures."""
        for start_sig, embedded_sig in self.POLYGLOT_SIGNATURES:
            if content.startswith(start_sig) and embedded_sig in content:
                return True
        return False

    def _has_double_extension(self, filename: str) -> bool:
        """Check for suspicious double extensions."""
        dangerous_patterns = [
            ".jpg.exe", ".png.exe", ".pdf.exe",
            ".doc.exe", ".txt.exe", ".js.txt",
        ]
        lower_name = filename.lower()
        return any(pattern in lower_name for pattern in dangerous_patterns)

# Usage
validator = SecureFileValidator()
with open("uploaded_file.jpg", "rb") as f:
    result = validator.validate_secure(f.read(), "uploaded_file.jpg")

if not result["secure"]:
    print(f"Security issues: {result['issues']}")

Content Security Pipeline

⤢ zoom

Best Practices

import magic
import hashlib

def secure_file_handling(content: bytes, filename: str) -> dict:
    """Comprehensive secure file handling."""

    mime = magic.Magic(mime=True)

    # 1. Always validate content, not just extension
    actual_type = mime.from_buffer(content)

    # 2. Use allowlist, not blocklist
    ALLOWED_TYPES = {"image/jpeg", "image/png", "application/pdf"}
    if actual_type not in ALLOWED_TYPES:
        raise ValueError(f"Type {actual_type} not in allowlist")

    # 3. Generate safe filename
    file_hash = hashlib.sha256(content).hexdigest()[:16]
    safe_ext = {
        "image/jpeg": ".jpg",
        "image/png": ".png",
        "application/pdf": ".pdf",
    }.get(actual_type, ".bin")
    safe_filename = f"{file_hash}{safe_ext}"

    # 4. Set appropriate content type headers
    content_disposition = f'attachment; filename="{safe_filename}"'

    return {
        "safe_filename": safe_filename,
        "content_type": actual_type,
        "content_disposition": content_disposition,
        "hash": file_hash,
    }

Performance Considerations

Optimising Magic Instance Usage

import magic
from functools import lru_cache

# BAD: Creating new instance per call
def detect_type_slow(filepath):
    return magic.Magic(mime=True).from_file(filepath)

# GOOD: Reuse instance
_mime_detector = magic.Magic(mime=True)

def detect_type_fast(filepath):
    return _mime_detector.from_file(filepath)

# GOOD: Cached instance factory
@lru_cache(maxsize=4)
def get_magic(mime=False, mime_encoding=False):
    return magic.Magic(mime=mime, mime_encoding=mime_encoding)

def detect_with_cache(filepath):
    return get_magic(mime=True).from_file(filepath)

Buffer Size Optimisation

import magic

# For large files, only read header
MAGIC_HEADER_SIZE = 1024  # Usually sufficient

def detect_large_file(filepath):
    """Efficient detection for large files."""
    mime = magic.Magic(mime=True)

    with open(filepath, "rb") as f:
        header = f.read(MAGIC_HEADER_SIZE)

    return mime.from_buffer(header)

# Batch processing with connection reuse
def detect_batch(filepaths):
    """Process multiple files efficiently."""
    mime = magic.Magic(mime=True)  # Single instance

    results = {}
    for filepath in filepaths:
        try:
            results[filepath] = mime.from_file(filepath)
        except Exception as e:
            results[filepath] = f"Error: {e}"

    return results

Parallel Processing

import magic
from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor
from pathlib import Path

# Thread-safe detection (each thread gets own instance)
def detect_file(filepath):
    """Thread-safe file detection."""
    mime = magic.Magic(mime=True)
    return filepath, mime.from_file(filepath)

def detect_parallel(filepaths, max_workers=4):
    """Parallel file type detection."""
    results = {}

    with ThreadPoolExecutor(max_workers=max_workers) as executor:
        futures = [executor.submit(detect_file, fp) for fp in filepaths]

        for future in futures:
            filepath, mime_type = future.result()
            results[filepath] = mime_type

    return results

# Usage
files = list(Path("/uploads").glob("*"))
results = detect_parallel(files)

Performance Comparison

Approach	Time per File	Memory	Notes
New instance per call	~1ms	High	Avoid in loops
Reused instance	~0.1ms	Low	Recommended
Buffer detection (1KB)	~0.05ms	Minimal	Best for large files
Parallel (4 workers)	~0.025ms	Medium	Best for batches

Quick Reference

Common Operations

import magic

# Basic usage
magic.from_file("file.pdf")              # Description
magic.from_file("file.pdf", mime=True)   # MIME type
magic.from_buffer(data)                  # From bytes
magic.from_buffer(data, mime=True)       # MIME from bytes

# With Magic class (recommended for reuse)
m = magic.Magic()                        # Description mode
m = magic.Magic(mime=True)               # MIME mode
m = magic.Magic(mime_encoding=True)      # Encoding mode
m = magic.Magic(mime=True, mime_encoding=True)  # Combined

# Custom database
m = magic.Magic(magic_file="/path/to/custom.magic")

Common MIME Types

File Type	MIME Type
JPEG	image/jpeg
PNG	image/png
GIF	image/gif
WebP	image/webp
PDF	application/pdf
JSON	application/json
HTML	text/html
Plain text	text/plain
ZIP	application/zip
GZIP	application/gzip
MP4	video/mp4
MP3	audio/mpeg

Validation Checklist

# Essential validation steps
def validate_upload(content: bytes, filename: str):
    mime = magic.Magic(mime=True)

    # 1. Size check
    assert len(content) <= MAX_SIZE

    # 2. Type allowlist
    mime_type = mime.from_buffer(content)
    assert mime_type in ALLOWED_TYPES

    # 3. Extension match
    ext = Path(filename).suffix.lower()
    assert ext in EXPECTED_EXTENSIONS[mime_type]

    return mime_type

Common Issues and Solutions

Installation Issues

Issue	Solution
MagicException: could not find any magic files	Install libmagic: apt-get install libmagic1 (Ubuntu) or brew install libmagic (macOS)
ImportError: failed to find libmagic	Install system library or use python-magic-bin on Windows
ModuleNotFoundError: No module named 'magic'	pip install python-magic (not pip install magic)
Conflict with filemagic package	Uninstall conflicting packages: pip uninstall filemagic python-magic && pip install python-magic

Detection Issues

Issue	Solution
Returns application/octet-stream for known type	File may be corrupted or use unusual encoding; check file integrity
Incorrect MIME type detection	Update libmagic database: apt-get update && apt-get upgrade libmagic1
Different results on different systems	Libmagic versions differ; pin version or use custom magic file
Empty file returns error	Add size check before detection

Performance Issues

Issue	Solution
Slow when processing many files	Reuse Magic instance; don't create new one per file
High memory usage	Read only file header (first 1KB) for large files
Thread safety problems	Create separate Magic instance per thread

Code Examples for Common Fixes

import magic

# Fix: Handle empty files
def safe_detect(filepath_or_buffer):
    mime = magic.Magic(mime=True)

    if isinstance(filepath_or_buffer, bytes):
        if len(filepath_or_buffer) == 0:
            return "application/x-empty"
        return mime.from_buffer(filepath_or_buffer)
    else:
        import os
        if os.path.getsize(filepath_or_buffer) == 0:
            return "application/x-empty"
        return mime.from_file(filepath_or_buffer)

# Fix: Handle encoding errors
def detect_with_fallback(content: bytes):
    mime = magic.Magic(mime=True)

    try:
        return mime.from_buffer(content)
    except magic.MagicException:
        # Fallback to octet-stream
        return "application/octet-stream"

# Fix: Cross-platform compatibility
def get_mime_detector():
    """Get platform-appropriate magic instance."""
    import platform

    if platform.system() == "Windows":
        # Windows might need explicit magic file path
        import os
        magic_file = os.environ.get("MAGIC")
        if magic_file:
            return magic.Magic(mime=True, magic_file=magic_file)

    return magic.Magic(mime=True)

Debugging Tips

import magic

# Check libmagic version
print(magic.version())  # Returns version number

# Verbose detection for debugging
def debug_detect(filepath):
    """Get detailed detection information."""
    m = magic.Magic()
    mime = magic.Magic(mime=True)
    encoding = magic.Magic(mime_encoding=True)

    with open(filepath, "rb") as f:
        header = f.read(32)

    print(f"File: {filepath}")
    print(f"Header hex: {header.hex()}")
    print(f"Description: {m.from_file(filepath)}")
    print(f"MIME: {mime.from_file(filepath)}")
    print(f"Encoding: {encoding.from_file(filepath)}")

# Test magic database
def test_magic_database():
    """Verify magic database is working."""
    # Use buffers libmagic can actually match. A bare 8-byte PNG signature
    # is not enough - include the IHDR chunk tag. ZIP detection needs the
    # end-of-central-directory record near the file's tail, which from_buffer
    # on a short prefix never sees (use from_file for ZIP), so test gzip/GIF
    # instead, which match from a leading buffer.
    test_cases = [
        (b"%PDF-1.4", "application/pdf"),
        (b"\x89PNG\r\n\x1a\n\x00\x00\x00\x0dIHDR", "image/png"),
        (b"\xff\xd8\xff\xe0", "image/jpeg"),
        (b"\x1f\x8b\x08\x00", "application/gzip"),
        (b"GIF89a", "image/gif"),
    ]

    mime = magic.Magic(mime=True)

    for data, expected in test_cases:
        result = mime.from_buffer(data)
        status = "OK" if result == expected else "FAIL"
        print(f"{status}: {expected} -> {result}")