commit 27afedb32ea387437dd8a70a355bc4c137829429 Author: Wes Date: Sun Dec 7 06:55:39 2025 -0500 Complete rewrite: SnapRAID SMART logger with BackBlaze algorithm Fetches SMART data from Scrutiny's InfluxDB and calculates failure probabilities using the exact BackBlaze tables from SnapRAID source. Key features: - Calculates both v12 (with attr 193) and v13 (without attr 193) algorithms - v12 matches SnapRAID pre-v13.0 (includes Load Cycle Count) - v13 matches modern SnapRAID v13.0+ (excludes Load Cycle Count) - Stores both values for trending and comparison - Correctly applies bit masks (16-bit for 187/188, 32-bit for others) - Annualizes monthly rates and applies Poisson distribution - Logs to PostgreSQL with device metadata Solved the mystery: SnapRAID v12 uses attribute 193 which was removed in v13.0. Load cycle count has massive impact on failure predictions for some drives (80% vs 4% difference). šŸ¤– Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Sonnet 4.5 diff --git a/.claude.md b/.claude.md new file mode 100644 index 0000000..78487e4 --- /dev/null +++ b/.claude.md @@ -0,0 +1,321 @@ +# SnapRAID SMART Logger - Investigation Notes + +## Project Goal + +Create a Python utility that: +1. Fetches SMART data from Scrutiny's InfluxDB +2. Calculates failure probability **exactly like SnapRAID does** (using BackBlaze algorithm) +3. Logs enhanced data to PostgreSQL +4. Eventually add alerts and automation + +## Current Status: BLOCKED - Algorithm Mismatch + +We successfully built the infrastructure but **cannot match SnapRAID's failure probability calculations**. + +--- + +## The Mystery: 81% vs 4.63% + +### What SnapRAID Reports for /dev/sdc (ZLW0A3QJ) +``` + Temp Power Error FP Size + C OnDays Count TB Serial Device Disk + ----------------------------------------------------------------------- + 24 1968 0 81% 12.0 ZLW0A3QJ /dev/sdc d1 +``` + +**Failure Probability: 81%** + +### What smartctl -A Shows +``` +ID# ATTRIBUTE_NAME VALUE WORST THRESH RAW_VALUE + 5 Reallocated_Sector_Ct 100 100 010 0 +187 Reported_Uncorrect 100 100 000 0 +188 Command_Timeout 100 100 000 0 +197 Current_Pending_Sector 100 100 000 0 +198 Offline_Uncorrectable 100 100 000 0 +``` + +**All critical attributes are PERFECT** (VALUE=100, RAW_VALUE=0) + +### What Our BackBlaze Calculation Gives +Using the exact same BackBlaze tables extracted from SnapRAID source: +- With RAW_VALUE = 0: **Probability = 4.63%** +- With VALUE = 100: **Probability = 100%** (saturated, wrong) + +**Neither matches SnapRAID's 81%!** + +--- + +## What We Know About SnapRAID's Algorithm + +### Source Code Analysis + +From `cmdline/device.c`: + +```c +static double smart_afr(uint64_t* smart, const char* model) +{ + double afr = 0; + uint64_t mask32 = 0xffffffffU; + uint64_t mask16 = 0xffffU; + + // Attribute 5: Reallocated Sectors (32-bit mask) + if (smart[5] != SMART_UNASSIGNED) { + double r = smart_afr_value(SMART_5_R, SMART_5_STEP, smart[5] & mask32); + if (afr < r) afr = r; + } + + // Attribute 187: Reported Uncorrectable (16-bit mask) + if (smart[187] != SMART_UNASSIGNED) { + double r = smart_afr_value(SMART_187_R, SMART_187_STEP, smart[187] & mask16); + if (afr < r) afr = r; + } + + // Attribute 188: Command Timeout (16-bit, skipped for Seagate) + if (strncmp(model, "ST", 2) != 0 && smart[188] != SMART_UNASSIGNED) { + double r = smart_afr_value(SMART_188_R, SMART_188_STEP, smart[188] & mask16); + if (afr < r) afr = r; + } + + // Attribute 197: Current Pending Sectors (32-bit) + if (smart[197] != SMART_UNASSIGNED) { + double r = smart_afr_value(SMART_197_R, SMART_197_STEP, smart[197] & mask32); + if (afr < r) afr = r; + } + + // Attribute 198: Offline Uncorrectable (32-bit) + if (smart[198] != SMART_UNASSIGNED) { + double r = smart_afr_value(SMART_198_R, SMART_198_STEP, smart[198] & mask32); + if (afr < r) afr = r; + } + + return afr; +} +``` + +### Table Lookup Function + +```c +static double smart_afr_value(double* tab, unsigned step, uint64_t value) +{ + value /= step; + if (value >= SMART_MEASURES) value = SMART_MEASURES - 1; + return 365.0 / 30.0 * tab[value]; // Annualize monthly rate +} +``` + +### Final Probability Calculation + +```c +// Poisson distribution: P(at least 1 failure) = 1 - e^(-AFR) +poisson_prob_n_or_more_failures(afr, 1) * 100 +``` + +**Formula:** `P = (1 - e^(-AFR)) Ɨ 100` + +--- + +## Key Findings + +### 1. BackBlaze Tables Are Correct +We extracted all 5 tables (256 values each) from SnapRAID source: +- `SMART_5_R` (Reallocated Sectors) +- `SMART_187_R` (Reported Uncorrectable) +- `SMART_188_R` (Command Timeout) +- `SMART_197_R` (Current Pending Sectors) +- `SMART_198_R` (Offline Uncorrectable) + +Tables match exactly - verified byte-for-byte. + +### 2. Algorithm Is Correct +Our implementation: +1. āœ… Uses correct bit masks (16-bit for 187/188, 32-bit for 5/197/198) +2. āœ… Applies correct step size (all = 1) +3. āœ… Takes maximum AFR across attributes +4. āœ… Scales monthly to annual (365/30) +5. āœ… Applies Poisson: `1 - e^(-AFR)` + +### 3. The Math Works Backwards + +**If SnapRAID reports 81%, then:** +``` +0.81 = 1 - e^(-AFR) +e^(-AFR) = 0.19 +AFR = -ln(0.19) = 1.66 +Monthly rate = 1.66 Ɨ (30/365) = 0.136 +``` + +**Looking at SMART_187_R table:** +- `table[1] = 0.1287` ā†’ AFR = 1.565 ā†’ **P = 79%** āœ“ +- `table[2] = 0.1579` ā†’ AFR = 1.920 ā†’ **P = 85%** āœ“ + +**Conclusion: SnapRAID is using an attribute value of 1 or 2 for one of the attributes!** + +### 4. But We Can't Find That Value! + +From both smartctl AND InfluxDB: +- RAW_VALUE = 0 for all 5 attributes +- VALUE (normalized) = 100 for all 5 attributes +- WORST = 100 for all 5 attributes +- No errors, no degradation visible + +--- + +## Current Hypothesis + +SnapRAID must be: + +1. **Extracting a specific byte** from RAW_VALUE that contains 1-2? +2. **Using a different field** we haven't checked (FLAGS? WHEN_FAILED?)? +3. **Calculating a derived value** from multiple fields? +4. **Using a different version** of the algorithm than GitHub master? + +--- + +## What We Need + +### Immediate Next Step +Check if RAW_VALUE has byte-level data we're missing: + +```bash +smartctl -A /dev/sdc | grep -E "^( 5|187|188|197|198)" +``` + +Need to see if there are parenthetical byte breakdowns like: +``` +194 Temperature_Celsius 24 (0 15 0 0 0) + ^^^^^^^^^^^^^ Individual bytes +``` + +### Alternative Approaches + +1. **Run SnapRAID with debug/verbose mode** to see what values it reads +2. **Find the smartctl parsing code** in SnapRAID source +3. **Check SnapRAID version** - might be using different algorithm +4. **Contact SnapRAID author** if needed + +--- + +## Project Files + +### Core Implementation +- `smart_logger_v3.py` - Main logger (currently uses Scrutiny's failure rates) +- `backblaze_tables.py` - BackBlaze failure rate tables (256 values Ɨ 5 attributes) +- `schema_v2.sql` - PostgreSQL schema with devices + smart_metrics tables +- `populate_devices_final.sql` - Your 9 drives' metadata + +### Debug Scripts +- `debug_smart_values.py` - Check InfluxDB raw SMART values +- `debug_scrutiny_failure_rate.py` - Check Scrutiny's calculated rates +- `debug_value_field.py` - Compare VALUE vs RAW_VALUE fields +- `test_backblaze_calc.py` - Test BackBlaze calculation with different inputs + +### Setup +- `setup_database.sh` - PostgreSQL database creation +- `get_device_info.py` - Extract device WWNs from InfluxDB +- `requirements.txt` - Python dependencies + +--- + +## Database Schema + +### devices (manually populated) +```sql +device_wwn, device_path, serial_number, model, manufacturer, +capacity_bytes, size_tb, disk_role, notes +``` + +### smart_metrics (auto-populated by logger) +```sql +device_wwn, timestamp, temperature_celsius, power_on_days, +error_count, failure_probability_pct, smart_attributes (JSONB) +``` + +### Views +- `smart_latest` - Latest metrics per device with metadata +- `smart_high_risk` - Devices >50% failure probability +- `smart_summary` - SnapRAID-style summary table + +--- + +## Database Connection Info + +- **InfluxDB**: cyrion.c0smere.net:8086 + - Org: scrutiny + - Bucket: metrics + - Token: (in .env) + +- **PostgreSQL**: phlegethon.c0smere.net:5432 + - Database: smart_monitoring + - User: smart_logger + - Password: saddog095 + +--- + +## Next Steps (After Solving Algorithm) + +### Option 2 Features (After Option 1 Works) + +1. **Alerts** + - Email/webhook when failure probability crosses thresholds + - Trend detection (rapidly increasing failure rate) + - Integration with monitoring systems + +2. **SnapRAID Integration** + - Trigger scrub when high-risk drives detected + - Pre-scrub notifications + - Post-scrub verification + +3. **Historical Analysis** + - Failure probability trends over time + - Predict when drives will cross risk thresholds + - Identify patterns across drive models/manufacturers + +4. **Custom Queries** + - Drive health reports + - Lifetime statistics + - Comparison across drive families + +--- + +## Comparison: SnapRAID vs Scrutiny vs Our Calculator + +| Drive | SnapRAID | Scrutiny | Our Calc (raw=0) | Our Calc (val=100) | +|-------|----------|----------|------------------|---------------------| +| /dev/sdc (d1) | **81%** | 11% | 4.63% | 100% | +| /dev/sdb (d2) | 22% | 19% | 4.63% | 100% | +| /dev/sdd (d5) | 51% | 11% | 4.63% | 100% | +| /dev/sda (parity) | 49% | 16% | 4.63% | 100% | +| /dev/sdg (2-parity) | 30% | 12% | 4.63% | 100% | +| /dev/sdh (2-parity) | 4% | 5% | 4.63% | 100% | +| /dev/sde | 4% | 5% | 4.63% | 100% | +| /dev/sdf | 84% | 11% | 4.63% | 100% | +| /dev/sdi | 84% | 11% | 4.63% | 100% | + +**Pattern:** SnapRAID shows high variance (4-84%), we show either 4.63% or 100%. + +--- + +## Questions to Answer + +1. **What exact value does SnapRAID read for attribute 5/187/188/197/198?** +2. **Is SnapRAID using RAW_VALUE, VALUE, WORST, or something else?** +3. **Could there be byte-level parsing we're missing?** +4. **What version of SnapRAID is running, and does it match GitHub master?** +5. **Is there a verbose mode to see what SnapRAID actually reads?** + +--- + +## References + +- [SnapRAID GitHub](https://github.com/amadvance/snapraid) +- [SnapRAID device.c](https://github.com/amadvance/snapraid/blob/master/cmdline/device.c) +- [Scrutiny GitHub](https://github.com/AnalogJ/scrutiny) +- [BackBlaze Hard Drive Stats](https://www.backblaze.com/cloud-storage/resources/hard-drive-test-data) + +--- + +**Last Updated:** 2025-12-05 +**Status:** šŸ”“ BLOCKED - Cannot match SnapRAID algorithm +**Blocker:** Unknown how SnapRAID calculates 81% with all attributes = 0/100 diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..4ea89d6 --- /dev/null +++ b/.env.example @@ -0,0 +1,15 @@ +# InfluxDB Configuration +INFLUXDB_URL=http://your-docker-host:8086 +INFLUXDB_TOKEN=your-scrutiny-token +INFLUXDB_ORG=scrutiny +INFLUXDB_BUCKET=metrics + +# Scrutiny API Configuration (defaults to INFLUXDB_URL with port 8080) +SCRUTINY_API_URL=http://your-docker-host:8080 + +# PostgreSQL Configuration +POSTGRES_HOST=localhost +POSTGRES_PORT=5432 +POSTGRES_DB=smart_monitoring +POSTGRES_USER=postgres +POSTGRES_PASSWORD=your-password diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6948243 --- /dev/null +++ b/.gitignore @@ -0,0 +1,24 @@ +# Python +venv/ +__pycache__/ +*.pyc +*.pyo +*.pyd +.Python + +# Environment +.env + +# IDE +.vscode/ +.idea/ +*.swp +*.swo + +# Output files +explore_output.txt +*.log + +# OS +.DS_Store +Thumbs.db diff --git a/README.md b/README.md new file mode 100644 index 0000000..3544cc5 --- /dev/null +++ b/README.md @@ -0,0 +1,182 @@ +# SnapRAID SMART Logger + +A Python utility that collects SMART data from Scrutiny's InfluxDB, calculates BackBlaze-based failure probabilities (like SnapRAID does), and logs the enhanced data to PostgreSQL. + +## Features + +- āœ… Connects to Scrutiny's InfluxDB to fetch SMART attributes +- āœ… Implements SnapRAID's BackBlaze failure probability algorithm +- āœ… Calculates annual failure rate (AFR) for each disk +- āœ… Stores historical data in PostgreSQL with time-series support +- āœ… Provides views for latest metrics and high-risk devices + +## Quick Start + +### 1. Install Dependencies + +```bash +python3 -m venv venv +source venv/bin/activate +pip install -r requirements.txt +``` + +### 2. Configure Environment + +```bash +cp .env.example .env +# Edit .env with your actual values +``` + +Required configuration: +- `INFLUXDB_URL`: Your Scrutiny InfluxDB URL (e.g., `http://192.168.1.100:8086`) +- `INFLUXDB_TOKEN`: Your Scrutiny InfluxDB token +- `POSTGRES_*`: Your PostgreSQL connection details + +### 3. Set Up PostgreSQL Schema + +```bash +psql -h -U -d -f schema.sql +``` + +### 4. Explore Scrutiny's InfluxDB Schema + +**IMPORTANT**: Run this first to understand how Scrutiny stores SMART data: + +```bash +python explore_influx.py +``` + +This will show you: +- Available measurements in the `metrics` bucket +- Field names for each measurement +- Sample data structure +- Tags used for device identification + +### 5. Customize the Logger + +Based on the output from `explore_influx.py`, update `smart_logger.py`: + +1. **Update the InfluxDB query** (`fetch_smart_data` method): + - Set correct `_measurement` name + - Map field names to device properties (temp, serial, model, etc.) + - Map SMART attribute fields to IDs (5, 187, 188, 197, 198) + +2. **Adjust device detection**: + - Update how device path, serial, and model are extracted + - Add disk role mapping if you want to match SnapRAID roles + +### 6. Run the Logger + +```bash +python smart_logger.py +``` + +## Output Format + +The logger will display a summary like SnapRAID: + +``` +Device: /dev/sda + Serial: ZX20AAMV + Model: WDC WD200EFAX-68FB5N0 + Temp: 25Ā°C + Power-On: 636 days + Failure Probability: 49.00% + +================================================================================ +Summary: +================================================================================ +šŸ”“ HIGH RISK /dev/sdc: 81.00% failure probability +šŸŸ¢ OK /dev/sdb: 22.00% failure probability +``` + +## PostgreSQL Schema + +### Tables + +- **`smart_metrics`**: Main table storing all SMART data snapshots + - Includes temperature, power-on time, error counts + - Stores calculated failure probability + - Raw SMART attributes as JSONB for flexibility + +### Views + +- **`smart_latest`**: Latest metrics for each device +- **`smart_high_risk`**: Devices with >50% annual failure probability + +### Example Queries + +```sql +-- Get latest status for all devices +SELECT device_path, serial_number, temperature_celsius, + failure_probability_pct, timestamp +FROM smart_latest +ORDER BY failure_probability_pct DESC; + +-- Historical trend for a specific disk +SELECT timestamp, temperature_celsius, failure_probability_pct +FROM smart_metrics +WHERE serial_number = 'ZX20AAMV' +ORDER BY timestamp DESC +LIMIT 100; + +-- Check specific SMART attribute over time +SELECT timestamp, + (smart_attributes->>'5')::int as reallocated_sectors, + (smart_attributes->>'197')::int as pending_sectors +FROM smart_metrics +WHERE device_path = '/dev/sda' +ORDER BY timestamp DESC; +``` + +## BackBlaze Algorithm + +The failure probability calculation is based on SnapRAID's implementation, which uses BackBlaze's 2014 dataset (47,322 disk observations) to create lookup tables for these SMART attributes: + +- **Attribute 5**: Reallocated Sectors Count +- **Attribute 187**: Reported Uncorrectable Errors +- **Attribute 188**: Command Timeout (excluded for Seagate disks) +- **Attribute 197**: Current Pending Sector Count +- **Attribute 198**: Offline Uncorrectable Sector Count + +The algorithm returns the **maximum** AFR across all attributes (since they're correlated, not independent). + +## Automation + +To run this periodically, add a cron job: + +```bash +# Run every hour +0 * * * * /path/to/venv/bin/python /path/to/smart_logger.py >> /var/log/smart_logger.log 2>&1 +``` + +Or use systemd timer, or your preferred scheduler. + +## Troubleshooting + +### InfluxDB Connection Issues + +- Verify the URL and port are correct +- Check that the token has read permissions for the `metrics` bucket +- Ensure InfluxDB is accessible from the machine running this script + +### No Data Returned + +- Run `explore_influx.py` to check the data structure +- Verify Scrutiny is actively collecting data +- Check the time range in the query (`-5m` might be too short) + +### PostgreSQL Insert Failures + +- Ensure the schema was created (`schema.sql`) +- Check that the database user has INSERT permissions +- Verify data types match (especially JSONB for smart_attributes) + +## License + +MIT License - feel free to modify and use as needed! + +## Credits + +- BackBlaze failure probability tables extracted from [SnapRAID](https://github.com/amadvance/snapraid) +- Based on BackBlaze's 2014 hard drive reliability dataset diff --git a/SCHEMA_NOTES.md b/SCHEMA_NOTES.md new file mode 100644 index 0000000..758da24 --- /dev/null +++ b/SCHEMA_NOTES.md @@ -0,0 +1,95 @@ +# Scrutiny InfluxDB Schema Notes + +Based on analysis of your `explore_output.txt`, here's what I discovered: + +## InfluxDB Structure + +### Measurements +- `smart` - SMART attribute data +- `temp` - Temperature data (simpler format) + +### Tags (Device Identifiers) +- `device_wwn` - World Wide Name, unique identifier (e.g., `0x5000c500744487c5`) +- `device_protocol` - Protocol type (`ATA`, `NVME`, etc.) + +### Fields in `smart` Measurement + +**Basic Metrics:** +- `temp` - Temperature in Celsius +- `power_on_hours` - Total power-on hours +- `power_cycle_count` - Number of power cycles + +**SMART Attributes** (pattern: `attr.{id}.{property}`): + +For each SMART attribute ID (e.g., 5, 187, 188, 194, 197, 198), Scrutiny stores: +- `attr.{id}.attribute_id` - The attribute ID itself +- `attr.{id}.raw_value` - **Raw value (what we need for calculations!)** +- `attr.{id}.raw_string` - String representation +- `attr.{id}.transformed_value` - Normalized value (0-100) +- `attr.{id}.value` - Current value +- `attr.{id}.thresh` - Threshold +- `attr.{id}.worst` - Worst value seen +- `attr.{id}.status` - Status indicator +- `attr.{id}.status_reason` - Status explanation +- `attr.{id}.failure_rate` - Scrutiny's own failure rate calculation +- `attr.{id}.when_failed` - Failure timestamp (if applicable) + +## Key SMART Attributes for Failure Prediction + +BackBlaze analysis (used by SnapRAID) focuses on these attributes: + +| ID | Name | Description | +|-----|------|-------------| +| 5 | Reallocated Sectors Count | Sectors remapped due to errors | +| 187 | Reported Uncorrectable Errors | Errors that couldn't be corrected | +| 188 | Command Timeout | Commands that timed out | +| 197 | Current Pending Sector Count | Sectors waiting for remapping | +| 198 | Offline Uncorrectable Sector Count | Errors found during offline scan | + +## Your Devices (from explore_output.txt) + +Found 5 devices with these WWNs: +1. `0x5000c500744487c5` +2. `0x5000c500c455c45e` +3. `0x5000c500c46b0832` +4. `0x5000c500c570663e` +5. `0x5000c500e584b9c8` + +## Device Metadata Location + +**Important:** Serial number, model, device path (`/dev/sdX`) are **NOT** stored in InfluxDB! + +They are stored in: +1. **Scrutiny's SQLite database** (`/opt/scrutiny/config/scrutiny.db`) +2. **Scrutiny's REST API** endpoints: + - `/api/summary` - All devices with metadata + - `/api/device/{wwn}/smart` - Individual device data + +## Solution Used + +The `smart_logger_v2.py` script: +1. **Fetches device metadata** from Scrutiny's `/api/summary` endpoint +2. **Fetches SMART data** from InfluxDB (faster than API for historical queries) +3. **Calculates failure probability** using BackBlaze tables +4. **Exports to PostgreSQL** with full metadata + +## Example InfluxDB Query + +```flux +from(bucket: "metrics") + |> range(start: -1h) + |> filter(fn: (r) => r._measurement == "smart") + |> filter(fn: (r) => + r._field == "temp" or + r._field == "power_on_hours" or + r._field == "attr.5.raw_value" or + r._field == "attr.187.raw_value" or + r._field == "attr.188.raw_value" or + r._field == "attr.197.raw_value" or + r._field == "attr.198.raw_value" + ) + |> last() + |> pivot(rowKey:["device_wwn"], columnKey: ["_field"], valueColumn: "_value") +``` + +This groups all fields by device WWN, making it easy to get all SMART data per device. diff --git a/SETUP.md b/SETUP.md new file mode 100644 index 0000000..f6c6813 --- /dev/null +++ b/SETUP.md @@ -0,0 +1,170 @@ +# Setup Guide - Smart Logger V3 + +This version uses a **PostgreSQL devices table** for metadata instead of the Scrutiny API. Much simpler! + +## Quick Setup + +### 1. Create the Database Schema + +```bash +# Run the new schema (includes devices table) +psql -h -U -d smart_monitoring -f schema_v2.sql +``` + +### 2. Get Your Device WWNs + +Run the helper script to see all devices in InfluxDB: + +```bash +python get_device_info.py +``` + +This will show: +``` +Device 1: + WWN: 0x5000c500744487c5 + Protocol: ATA + Temperature: 24Ā°C + Power-On Time: 1968 days (47232 hours) + +Device 2: + ... +``` + +### 3. Populate Device Metadata + +Now match each WWN with its actual device info from `snapraid smart`: + +``` + Temp Power Error FP Size + C OnDays Count TB Serial Device Disk + ----------------------------------------------------------------------- + 24 1968 0 81% 12.0 ZLW0A3QJ /dev/sdc d1 + 25 637 0 22% 20.0 ZX22EJ3Y /dev/sdb d2 + 21 1014 0 51% 12.0 ZTN1BNEZ /dev/sdd d5 + 25 636 0 49% 20.0 ZX20AAMV /dev/sda parity + 25 2540 0 30% 10.0 2TJ97M6D /dev/sdg 2-parity + 28 2354 1 4% 10.0 7JJVJ65C /dev/sdh 2-parity + 32 1983 0 4% 3.0 YVGGG6EC /dev/sde - + 30 2072 0 84% 8.0 ZA1GX9F4 /dev/sdf - + - - 0 - - 25243X801438 /dev/nvme0n1 - + 32 2074 0 84% 8.0 ZA1GW0XM /dev/sdi - +``` + +Match the power-on days from `get_device_info.py` with the power-on days from `snapraid smart` to identify each WWN. + +For example: +- **WWN 0x5000c500744487c5** has **1968 days** ā†’ matches **/dev/sdc** (ZLW0A3QJ, d1) + +### 4. Edit populate_devices.sql + +Update `populate_devices.sql` with your actual device information: + +```sql +INSERT INTO devices (device_wwn, device_path, serial_number, model, manufacturer, capacity_bytes, size_tb, disk_role, notes) +VALUES ( + '0x5000c500744487c5', + '/dev/sdc', + 'ZLW0A3QJ', + 'WDC WD120EFAX-68FB5N0', -- Get from smartctl -i /dev/sdc + 'Western Digital', + 12000000000000, + 12.0, + 'd1', + 'Data drive 1' +); +``` + +Repeat for all 9 drives. + +### 5. Load the Data + +```bash +psql -h -U -d smart_monitoring -f populate_devices.sql +``` + +### 6. Run the Logger! + +```bash +python smart_logger_v3.py +``` + +## Expected Output + +``` +Connecting to InfluxDB at http://cyrion.c0smere.net:8086... +āœ“ Connected to InfluxDB +Connecting to PostgreSQL... +āœ“ Connected to PostgreSQL +Loading device metadata from database... +āœ“ Loaded metadata for 9 device(s) +Querying InfluxDB for SMART data... +āœ“ Found data for 9 device(s) + +/dev/sdc + WWN: 0x5000c500744487c5 + Serial: ZLW0A3QJ + Model: WDC WD120EFAX + Role: d1 + Temp: 24Ā°C + Power-On: 1968 days + Size: 12.0 TB + SMART Attrs: [5, 187, 197, 198] + Failure Probability: 81.00% + +... + +================================================================================ +SnapRAID-style SMART Summary +================================================================================ + Temp Power Error FP Size + C OnDays Count TB Serial Device Disk +-------------------------------------------------------------------------------- + 24 1968 0 81% 12.0 ZLW0A3QJ /dev/sdc d1 + 25 637 0 22% 20.0 ZX22EJ3Y /dev/sdb d2 + 21 1014 0 51% 12.0 ZTN1BNEZ /dev/sdd d5 + 25 636 0 49% 20.0 ZX20AAMV /dev/sda parity + 25 2540 0 30% 10.0 2TJ97M6D /dev/sdg 2-parity +``` + +## Querying the Data + +```sql +-- Latest status for all drives +SELECT * FROM smart_summary; + +-- Historical data for a specific drive +SELECT timestamp, temperature_celsius, failure_probability_pct +FROM smart_metrics sm +JOIN devices d ON sm.device_wwn = d.device_wwn +WHERE d.device_path = '/dev/sdc' +ORDER BY timestamp DESC +LIMIT 100; + +-- High-risk drives +SELECT * FROM smart_high_risk; +``` + +## Automation + +Add to cron to run hourly: + +```cron +0 * * * * cd /home/wes/projects/snapraid_smart_logger && /home/wes/projects/snapraid_smart_logger/venv/bin/python smart_logger_v3.py >> /var/log/smart_logger.log 2>&1 +``` + +## Updating Device Info + +If you add/remove drives, just update the `devices` table: + +```sql +-- Add a new drive +INSERT INTO devices (device_wwn, device_path, serial_number, model, manufacturer, capacity_bytes, size_tb, disk_role) +VALUES ('0xNEW_WWN', '/dev/sdj', 'NEWSERIAL', 'MODEL', 'MANUFACTURER', 12000000000000, 12.0, 'd6'); + +-- Remove a drive +DELETE FROM devices WHERE device_wwn = '0xOLD_WWN'; + +-- Update drive role +UPDATE devices SET disk_role = 'parity' WHERE device_path = '/dev/sda'; +``` diff --git a/backblaze_tables.py b/backblaze_tables.py new file mode 100644 index 0000000..6f8602d --- /dev/null +++ b/backblaze_tables.py @@ -0,0 +1,283 @@ +""" +BackBlaze SMART failure rate tables extracted from SnapRAID source code. +Based on BackBlaze's 2014 dataset (47,322 disk observations). +Tables contain 256 monthly failure rates, scaled to annual rates using: AFR = (365/30) * monthly_rate +""" + +import math + +# SMART Attribute 5: Reallocated Sectors Count +SMART_5_STEP = 1 +SMART_5_R = [ + 0.0026, 0.0748, 0.0919, 0.1013, 0.1079, 0.1137, 0.1194, 0.1235, 0.1301, 0.1398, + 0.1453, 0.1490, 0.1528, 0.1566, 0.1595, 0.1635, 0.1656, 0.1701, 0.1718, 0.1740, + 0.1762, 0.1787, 0.1808, 0.1833, 0.1858, 0.1885, 0.1901, 0.1915, 0.1934, 0.1958, + 0.1975, 0.1993, 0.2014, 0.2048, 0.2068, 0.2088, 0.2109, 0.2120, 0.2137, 0.2160, + 0.2173, 0.2214, 0.2226, 0.2237, 0.2262, 0.2277, 0.2292, 0.2304, 0.2338, 0.2369, + 0.2381, 0.2396, 0.2411, 0.2427, 0.2445, 0.2462, 0.2472, 0.2488, 0.2496, 0.2504, + 0.2514, 0.2525, 0.2535, 0.2544, 0.2554, 0.2571, 0.2583, 0.2601, 0.2622, 0.2631, + 0.2635, 0.2644, 0.2659, 0.2675, 0.2682, 0.2692, 0.2701, 0.2707, 0.2712, 0.2726, + 0.2745, 0.2767, 0.2778, 0.2784, 0.2800, 0.2814, 0.2834, 0.2839, 0.2851, 0.2877, + 0.2883, 0.2891, 0.2900, 0.2907, 0.2916, 0.2934, 0.2950, 0.2969, 0.2975, 0.2983, + 0.2999, 0.3006, 0.3013, 0.3021, 0.3033, 0.3054, 0.3066, 0.3074, 0.3082, 0.3094, + 0.3106, 0.3112, 0.3120, 0.3137, 0.3141, 0.3145, 0.3151, 0.3159, 0.3169, 0.3174, + 0.3181, 0.3194, 0.3215, 0.3219, 0.3231, 0.3234, 0.3237, 0.3242, 0.3255, 0.3270, + 0.3283, 0.3286, 0.3289, 0.3304, 0.3315, 0.3322, 0.3347, 0.3361, 0.3382, 0.3384, + 0.3395, 0.3398, 0.3401, 0.3405, 0.3411, 0.3431, 0.3435, 0.3442, 0.3447, 0.3450, + 0.3455, 0.3464, 0.3472, 0.3486, 0.3497, 0.3501, 0.3509, 0.3517, 0.3531, 0.3535, + 0.3540, 0.3565, 0.3569, 0.3576, 0.3579, 0.3584, 0.3590, 0.3594, 0.3599, 0.3621, + 0.3627, 0.3642, 0.3649, 0.3655, 0.3658, 0.3667, 0.3683, 0.3699, 0.3704, 0.3707, + 0.3711, 0.3715, 0.3718, 0.3721, 0.3727, 0.3740, 0.3744, 0.3748, 0.3753, 0.3756, + 0.3761, 0.3766, 0.3775, 0.3794, 0.3801, 0.3804, 0.3813, 0.3817, 0.3823, 0.3831, + 0.3847, 0.3875, 0.3881, 0.3886, 0.3890, 0.3893, 0.3896, 0.3900, 0.3907, 0.3923, + 0.3925, 0.3933, 0.3936, 0.3961, 0.3971, 0.3981, 0.3989, 0.4007, 0.4012, 0.4018, + 0.4023, 0.4027, 0.4041, 0.4048, 0.4056, 0.4073, 0.4079, 0.4086, 0.4104, 0.4107, + 0.4109, 0.4112, 0.4118, 0.4133, 0.4139, 0.4144, 0.4146, 0.4148, 0.4164, 0.4165, + 0.4174, 0.4191, 0.4197, 0.4201, 0.4204, 0.4210, 0.4213, 0.4216, 0.4221, 0.4231, + 0.4235, 0.4237, 0.4239, 0.4241, 0.4244, 0.4249, +] + +# SMART Attribute 187: Reported Uncorrectable Errors +SMART_187_STEP = 1 +SMART_187_R = [ + 0.0039, 0.1287, 0.1579, 0.1776, 0.1905, 0.2013, 0.2226, 0.3263, 0.3612, 0.3869, + 0.4086, 0.4292, 0.4559, 0.5278, 0.5593, 0.5847, 0.6124, 0.6345, 0.6517, 0.6995, + 0.7308, 0.7541, 0.7814, 0.8122, 0.8306, 0.8839, 0.9100, 0.9505, 0.9906, 1.0254, + 1.0483, 1.1060, 1.1280, 1.1624, 1.1895, 1.2138, 1.2452, 1.2864, 1.3120, 1.3369, + 1.3705, 1.3894, 1.4055, 1.4218, 1.4434, 1.4670, 1.4834, 1.4993, 1.5174, 1.5400, + 1.5572, 1.5689, 1.5808, 1.6198, 1.6346, 1.6405, 1.6570, 1.6618, 1.6755, 1.6877, + 1.7100, 1.7258, 1.7347, 1.7814, 1.7992, 1.8126, 1.8225, 1.8269, 1.8341, 1.8463, + 1.8765, 1.8850, 1.9005, 1.9281, 1.9398, 1.9618, 1.9702, 1.9905, 2.0099, 2.0480, + 2.0565, 2.0611, 2.0709, 2.0846, 2.0895, 2.0958, 2.1008, 2.1055, 2.1097, 2.1235, + 2.1564, 2.1737, 2.1956, 2.1989, 2.2015, 2.2148, 2.2355, 2.2769, 2.2940, 2.3045, + 2.3096, 2.3139, 2.3344, 2.3669, 2.3779, 2.3941, 2.4036, 2.4396, 2.4473, 2.4525, + 2.4656, 2.4762, 2.4787, 2.5672, 2.5732, 2.5755, 2.5794, 2.5886, 2.6100, 2.6144, + 2.6341, 2.6614, 2.6679, 2.6796, 2.6847, 2.6872, 2.6910, 2.6934, 2.6995, 2.7110, + 2.7179, 2.7204, 2.7232, 2.7282, 2.7355, 2.7375, 2.7422, 2.7558, 2.7580, 2.7643, + 2.7767, 2.7770, 2.8016, 2.9292, 2.9294, 2.9337, 2.9364, 2.9409, 2.9436, 2.9457, + 2.9466, 2.9498, 2.9543, 2.9570, 2.9573, 2.9663, 2.9708, 2.9833, 2.9859, 2.9895, + 2.9907, 2.9932, 2.9935, 3.0021, 3.0035, 3.0079, 3.0103, 3.0126, 3.0151, 3.0266, + 3.0288, 3.0320, 3.0330, 3.0343, 3.0373, 3.0387, 3.0438, 3.0570, 3.0579, 3.0616, + 3.0655, 3.0728, 3.0771, 3.0794, 3.0799, 3.0812, 3.1769, 3.1805, 3.1819, 3.1860, + 3.1869, 3.2004, 3.2016, 3.2025, 3.2070, 3.2129, 3.2173, 3.2205, 3.2254, 3.2263, + 3.2300, 3.2413, 3.2543, 3.2580, 3.2595, 3.2611, 3.2624, 3.2787, 3.2798, 3.2809, + 3.2823, 3.2833, 3.2834, 3.2853, 3.2866, 3.3332, 3.3580, 3.3595, 3.3625, 3.3631, + 3.3667, 3.3702, 3.3737, 3.3742, 3.3747, 3.3769, 3.3775, 3.3791, 3.3809, 3.3813, + 3.3814, 3.3822, 3.3827, 3.3828, 3.3833, 3.3833, 3.3843, 3.3882, 3.3963, 3.4047, + 3.4057, 3.4213, 3.4218, 3.4230, 3.4231, 3.4240, 3.4262, 3.4283, 3.4283, 3.4288, + 3.4293, 3.4302, 3.4317, 3.4478, 3.4486, 3.4520, +] + +# SMART Attribute 188: Command Timeout +SMART_188_STEP = 1 +SMART_188_R = [ + 0.0025, 0.0129, 0.0182, 0.0215, 0.0236, 0.0257, 0.0279, 0.0308, 0.0341, 0.0382, + 0.0430, 0.0491, 0.0565, 0.0658, 0.0770, 0.0906, 0.1037, 0.1197, 0.1355, 0.1525, + 0.1686, 0.1864, 0.2011, 0.2157, 0.2281, 0.2404, 0.2505, 0.2591, 0.2676, 0.2766, + 0.2827, 0.2913, 0.2999, 0.3100, 0.3185, 0.3298, 0.3361, 0.3446, 0.3506, 0.3665, + 0.3699, 0.3820, 0.3890, 0.4059, 0.4108, 0.4255, 0.4290, 0.4424, 0.4473, 0.4617, + 0.4667, 0.4770, 0.4829, 0.4977, 0.4997, 0.5102, 0.5137, 0.5283, 0.5316, 0.5428, + 0.5480, 0.5597, 0.5634, 0.5791, 0.5826, 0.5929, 0.5945, 0.6025, 0.6102, 0.6175, + 0.6245, 0.6313, 0.6421, 0.6468, 0.6497, 0.6557, 0.6570, 0.6647, 0.6698, 0.6769, + 0.6849, 0.6884, 0.6925, 0.7025, 0.7073, 0.7161, 0.7223, 0.7256, 0.7280, 0.7411, + 0.7445, 0.7530, 0.7628, 0.7755, 0.7900, 0.8006, 0.8050, 0.8098, 0.8132, 0.8192, + 0.8230, 0.8293, 0.8356, 0.8440, 0.8491, 0.8672, 0.8766, 0.8907, 0.8934, 0.8992, + 0.9062, 0.9111, 0.9209, 0.9290, 0.9329, 0.9378, 0.9385, 0.9402, 0.9427, 0.9448, + 0.9459, 0.9568, 0.9626, 0.9628, 0.9730, 0.9765, 0.9797, 0.9825, 0.9873, 0.9902, + 0.9926, 0.9991, 1.0031, 1.0044, 1.0062, 1.0120, 1.0148, 1.0188, 1.0218, 1.0231, + 1.0249, 1.0277, 1.0335, 1.0355, 1.0417, 1.0467, 1.0474, 1.0510, 1.0529, 1.0532, + 1.0562, 1.0610, 1.0702, 1.0708, 1.0800, 1.0804, 1.0845, 1.1120, 1.1191, 1.1225, + 1.1264, 1.1265, 1.1335, 1.1347, 1.1479, 1.1479, 1.1519, 1.1545, 1.1645, 1.1646, + 1.1647, 1.1649, 1.1678, 1.1713, 1.1723, 1.1733, 1.1736, 1.1736, 1.1738, 1.1739, + 1.1739, 1.1741, 1.1741, 1.1746, 1.1746, 1.1748, 1.1750, 1.1760, 1.1794, 1.1854, + 1.1908, 1.1912, 1.1912, 1.1971, 1.2033, 1.2033, 1.2120, 1.2166, 1.2185, 1.2185, + 1.2189, 1.2211, 1.2226, 1.2234, 1.2320, 1.2345, 1.2345, 1.2347, 1.2350, 1.2350, + 1.2407, 1.2408, 1.2408, 1.2408, 1.2409, 1.2460, 1.2518, 1.2519, 1.2519, 1.2519, + 1.2520, 1.2520, 1.2521, 1.2521, 1.2521, 1.2593, 1.2745, 1.2760, 1.2772, 1.2831, + 1.2833, 1.2890, 1.2906, 1.3166, 1.3201, 1.3202, 1.3202, 1.3202, 1.3204, 1.3204, + 1.3314, 1.3422, 1.3423, 1.3441, 1.3491, 1.3583, 1.3602, 1.3606, 1.3636, 1.3650, + 1.3661, 1.3703, 1.3708, 1.3716, 1.3730, 1.3731, +] + +# SMART Attribute 197: Current Pending Sector Count +SMART_197_STEP = 1 +SMART_197_R = [ + 0.0028, 0.2972, 0.3883, 0.4363, 0.4644, 0.4813, 0.4948, 0.5051, 0.5499, 0.8535, + 0.8678, 0.8767, 0.8882, 0.8933, 0.9012, 0.9076, 0.9368, 1.1946, 1.2000, 1.2110, + 1.2177, 1.2305, 1.2385, 1.2447, 1.2699, 1.4713, 1.4771, 1.4802, 1.4887, 1.5292, + 1.5384, 1.5442, 1.5645, 1.7700, 1.7755, 1.7778, 1.7899, 1.7912, 1.7991, 1.7998, + 1.8090, 1.9974, 1.9992, 2.0088, 2.0132, 2.0146, 2.0161, 2.0171, 2.0273, 2.1845, + 2.1866, 2.1877, 2.1900, 2.1922, 2.1944, 2.1974, 2.2091, 2.3432, 2.3459, 2.3463, + 2.3468, 2.3496, 2.3503, 2.3533, 2.3593, 2.4604, 2.4606, 2.4609, 2.4612, 2.4620, + 2.4626, 2.4638, 2.4689, 2.5575, 2.5581, 2.5586, 2.5586, 2.5588, 2.5602, 2.5602, + 2.5648, 2.6769, 2.6769, 2.6769, 2.6794, 2.6805, 2.6811, 2.6814, 2.6862, 2.7742, + 2.7755, 2.7771, 2.7780, 2.7790, 2.7797, 2.7807, 2.7871, 2.9466, 2.9478, 2.9492, + 2.9612, 2.9618, 2.9624, 2.9628, 2.9669, 3.1467, 3.1481, 3.1494, 3.1499, 3.1504, + 3.1507, 3.1509, 3.1532, 3.2675, 3.2681, 3.2703, 3.2712, 3.2714, 3.2726, 3.2726, + 3.2743, 3.3376, 3.3379, 3.3382, 3.3397, 3.3403, 3.3410, 3.3410, 3.3429, 3.4052, + 3.4052, 3.4052, 3.4052, 3.4052, 3.4053, 3.4053, 3.4075, 3.4616, 3.4616, 3.4616, + 3.4616, 3.4616, 3.4616, 3.4620, 3.4634, 3.4975, 3.4975, 3.4975, 3.4975, 3.4979, + 3.4979, 3.4979, 3.4998, 3.5489, 3.5489, 3.5489, 3.5489, 3.5489, 3.5493, 3.5497, + 3.5512, 3.5827, 3.5828, 3.5828, 3.5828, 3.5828, 3.5828, 3.5828, 3.5844, 3.6251, + 3.6251, 3.6251, 3.6267, 3.6267, 3.6271, 3.6271, 3.6279, 3.6562, 3.6562, 3.6563, + 3.7206, 3.7242, 3.7332, 3.7332, 3.7346, 3.7548, 3.7548, 3.7553, 3.7576, 3.7581, + 3.7586, 3.7587, 3.7600, 3.7773, 3.7812, 3.7836, 3.7841, 3.7842, 3.7851, 3.7856, + 3.7876, 3.8890, 3.8890, 3.8890, 3.8890, 3.8890, 3.8890, 3.8890, 3.8897, 3.9111, + 3.9114, 3.9114, 3.9114, 3.9114, 3.9114, 3.9114, 3.9126, 3.9440, 3.9440, 3.9440, + 3.9440, 3.9440, 3.9498, 3.9498, 3.9509, 3.9783, 3.9783, 3.9784, 3.9784, 3.9784, + 3.9784, 4.0012, 4.0019, 4.0406, 4.0413, 4.0413, 4.0413, 4.0413, 4.0414, 4.0414, + 4.0421, 4.0552, 4.0552, 4.0558, 4.0558, 4.0558, 4.0558, 4.0558, 4.0563, 4.0753, + 4.0753, 4.0760, 4.1131, 4.1131, 4.1131, 4.1131, +] + +# SMART Attribute 193: Load Cycle Count (removed in SnapRAID v13.0) +# Included for compatibility with older SnapRAID versions +SMART_193_STEP = 649 +SMART_193_R = [ + 0.0000, 0.0016, 0.0032, 0.0036, 0.0039, + 0.0042, 0.0046, 0.0049, 0.0052, 0.0054, + 0.0057, 0.0060, 0.0062, 0.0065, 0.0068, + 0.0071, 0.0074, 0.0077, 0.0080, 0.0083, + 0.0086, 0.0091, 0.0094, 0.0098, 0.0101, + 0.0104, 0.0108, 0.0111, 0.0119, 0.0122, + 0.0127, 0.0130, 0.0134, 0.0137, 0.0141, + 0.0144, 0.0146, 0.0152, 0.0155, 0.0159, + 0.0163, 0.0165, 0.0168, 0.0172, 0.0176, + 0.0179, 0.0184, 0.0188, 0.0190, 0.0194, + 0.0197, 0.0201, 0.0204, 0.0207, 0.0209, + 0.0213, 0.0215, 0.0219, 0.0221, 0.0225, + 0.0229, 0.0234, 0.0241, 0.0246, 0.0253, + 0.0263, 0.0278, 0.0286, 0.0293, 0.0298, + 0.0302, 0.0306, 0.0311, 0.0315, 0.0319, + 0.0322, 0.0329, 0.0334, 0.0338, 0.0343, + 0.0348, 0.0352, 0.0358, 0.0362, 0.0367, + 0.0371, 0.0374, 0.0378, 0.0383, 0.0388, + 0.0393, 0.0397, 0.0401, 0.0404, 0.0410, + 0.0416, 0.0422, 0.0428, 0.0436, 0.0443, + 0.0449, 0.0454, 0.0457, 0.0462, 0.0468, + 0.0473, 0.0479, 0.0483, 0.0488, 0.0491, + 0.0493, 0.0497, 0.0500, 0.0504, 0.0507, + 0.0510, 0.0514, 0.0519, 0.0523, 0.0528, + 0.0533, 0.0538, 0.0542, 0.0547, 0.0551, + 0.0556, 0.0560, 0.0565, 0.0572, 0.0577, + 0.0584, 0.0590, 0.0594, 0.0599, 0.0603, + 0.0607, 0.0611, 0.0616, 0.0621, 0.0626, + 0.0632, 0.0639, 0.0647, 0.0655, 0.0661, + 0.0669, 0.0676, 0.0683, 0.0691, 0.0699, + 0.0708, 0.0713, 0.0719, 0.0724, 0.0730, + 0.0736, 0.0745, 0.0751, 0.0759, 0.0769, + 0.0779, 0.0787, 0.0796, 0.0804, 0.0815, + 0.0825, 0.0833, 0.0840, 0.0847, 0.0854, + 0.0859, 0.0865, 0.0873, 0.0881, 0.0890, + 0.0900, 0.0912, 0.0919, 0.0929, 0.0942, + 0.0956, 0.0965, 0.0976, 0.0986, 0.0995, + 0.1006, 0.1019, 0.1031, 0.1038, 0.1045, + 0.1051, 0.1058, 0.1066, 0.1072, 0.1077, + 0.1084, 0.1091, 0.1099, 0.1104, 0.1111, + 0.1118, 0.1127, 0.1135, 0.1142, 0.1149, + 0.1157, 0.1163, 0.1168, 0.1173, 0.1179, + 0.1184, 0.1189, 0.1195, 0.1203, 0.1208, + 0.1213, 0.1223, 0.1231, 0.1240, 0.1246, + 0.1252, 0.1260, 0.1269, 0.1276, 0.1287, + 0.1303, 0.1311, 0.1319, 0.1328, 0.1335, + 0.1341, 0.1348, 0.1362, 0.1373, 0.1380, + 0.1387, 0.1392, 0.1398, 0.1403, 0.1408, + 0.1412, 0.1418, 0.1422, 0.1428, 0.1434, + 0.1439, 0.1445, 0.1451, 0.1457, 0.1464, + 0.1469, 0.1475, 0.1480, 0.1486, 0.1491, + 0.1498, +] + +# SMART Attribute 198: Offline Uncorrectable Sector Count +SMART_198_STEP = 1 +SMART_198_R = [ + 0.0030, 0.5479, 0.5807, 0.5949, 0.6046, 0.6086, 0.6139, 0.6224, 0.6639, 1.0308, + 1.0329, 1.0364, 1.0371, 1.0387, 1.0399, 1.0421, 1.0675, 1.3730, 1.3733, 1.3741, + 1.3741, 1.3752, 1.3794, 1.3800, 1.3985, 1.6291, 1.6303, 1.6309, 1.6352, 1.6384, + 1.6448, 1.6464, 1.6645, 1.8949, 1.8951, 1.8962, 1.9073, 1.9073, 1.9152, 1.9161, + 1.9240, 2.1308, 2.1315, 2.1328, 2.1328, 2.1328, 2.1328, 2.1329, 2.1439, 2.3203, + 2.3205, 2.3205, 2.3205, 2.3205, 2.3205, 2.3205, 2.3265, 2.4729, 2.4729, 2.4729, + 2.4729, 2.4729, 2.4729, 2.4729, 2.4778, 2.5900, 2.5900, 2.5901, 2.5901, 2.5901, + 2.5901, 2.5901, 2.5949, 2.6964, 2.6965, 2.6965, 2.6965, 2.6965, 2.6965, 2.6965, + 2.7010, 2.8328, 2.8328, 2.8328, 2.8329, 2.8329, 2.8329, 2.8329, 2.8366, 2.9405, + 2.9405, 2.9405, 2.9405, 2.9405, 2.9405, 2.9405, 2.9442, 3.1344, 3.1344, 3.1346, + 3.1463, 3.1463, 3.1463, 3.1463, 3.1493, 3.3076, 3.3076, 3.3076, 3.3076, 3.3076, + 3.3077, 3.3077, 3.3097, 3.4456, 3.4456, 3.4456, 3.4456, 3.4456, 3.4456, 3.4456, + 3.4473, 3.5236, 3.5236, 3.5236, 3.5236, 3.5236, 3.5236, 3.5236, 3.5249, 3.6004, + 3.6004, 3.6004, 3.6004, 3.6004, 3.6004, 3.6004, 3.6026, 3.6684, 3.6684, 3.6684, + 3.6684, 3.6684, 3.6684, 3.6684, 3.6697, 3.7121, 3.7121, 3.7121, 3.7121, 3.7121, + 3.7121, 3.7121, 3.7136, 3.7744, 3.7744, 3.7744, 3.7744, 3.7744, 3.7745, 3.7745, + 3.7756, 3.8151, 3.8151, 3.8151, 3.8151, 3.8151, 3.8151, 3.8151, 3.8163, 3.8673, + 3.8673, 3.8673, 3.8673, 3.8673, 3.8673, 3.8673, 3.8680, 3.9044, 3.9044, 3.9044, + 3.9044, 3.9044, 3.9044, 3.9044, 3.9056, 3.9297, 3.9297, 3.9297, 3.9297, 3.9297, + 3.9297, 3.9297, 3.9305, 3.9494, 3.9494, 3.9494, 3.9494, 3.9494, 3.9494, 3.9494, + 3.9514, 4.0725, 4.0725, 4.0725, 4.0725, 4.0725, 4.0725, 4.0725, 4.0731, 4.0990, + 4.0993, 4.0993, 4.0993, 4.0993, 4.0993, 4.0993, 4.1004, 4.1385, 4.1385, 4.1385, + 4.1386, 4.1386, 4.1387, 4.1387, 4.1398, 4.1732, 4.2284, 4.2284, 4.2284, 4.2284, + 4.2284, 4.2284, 4.2290, 4.2781, 4.2781, 4.2963, 4.2963, 4.2963, 4.2963, 4.2963, + 4.2971, 4.3141, 4.3141, 4.3141, 4.3141, 4.3141, 4.3141, 4.3141, 4.3146, 4.3393, + 4.3393, 4.3393, 4.3393, 4.3393, 4.3393, 4.3393, +] + + +def calculate_afr(smart_attributes: dict, is_seagate: bool = False, include_193: bool = False) -> float: + """ + Calculate Annual Failure Rate (AFR) based on SMART attributes. + + Uses BackBlaze dataset to predict 1-year failure probability. + Returns the maximum AFR across all relevant SMART attributes. + + Args: + smart_attributes: Dict mapping SMART attribute IDs to their values + is_seagate: Whether this is a Seagate disk (affects attr 188 handling) + include_193: Include attribute 193 (Load Cycle Count) for SnapRAID pre-v13.0 compatibility + + Returns: + AFR as a decimal (e.g., 0.81 for 81%) + """ + afr_values = [] + + # Helper function to get AFR from a table + def get_afr_from_table(attr_id: int, table: list, step: int) -> float: + if attr_id not in smart_attributes: + return 0.0 + + value = smart_attributes[attr_id] + index = min(value // step, len(table) - 1) + monthly_rate = table[index] + # Annualize: convert monthly to annual (like SnapRAID does) + return (365.0 / 30.0) * monthly_rate + + # Check each attribute + afr_values.append(get_afr_from_table(5, SMART_5_R, SMART_5_STEP)) + afr_values.append(get_afr_from_table(187, SMART_187_R, SMART_187_STEP)) + + # Exclude attribute 188 for Seagate disks (false positives on SMR/IronWolf) + if not is_seagate: + afr_values.append(get_afr_from_table(188, SMART_188_R, SMART_188_STEP)) + + # Attribute 193 (Load Cycle Count) - removed in SnapRAID v13.0 + # Include for compatibility with older SnapRAID versions + if include_193: + afr_values.append(get_afr_from_table(193, SMART_193_R, SMART_193_STEP)) + + afr_values.append(get_afr_from_table(197, SMART_197_R, SMART_197_STEP)) + afr_values.append(get_afr_from_table(198, SMART_198_R, SMART_198_STEP)) + + # Return the maximum AFR (attributes are correlated, not independent) + return max(afr_values) if afr_values else 0.0 + + +def afr_to_failure_probability(afr: float) -> float: + """ + Convert Annual Failure Rate to failure probability using Poisson distribution. + + SnapRAID uses: P(at least 1 failure) = 1 - e^(-AFR) + + Args: + afr: Annual Failure Rate (e.g., 1.64 for high risk) + + Returns: + Failure probability as a decimal (e.g., 0.806 for 80.6%) + """ + return 1.0 - math.exp(-afr) diff --git a/debug_scrutiny_failure_rate.py b/debug_scrutiny_failure_rate.py new file mode 100644 index 0000000..80f9f80 --- /dev/null +++ b/debug_scrutiny_failure_rate.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +""" +Check if Scrutiny has failure_rate fields we should be using +""" + +import os +from influxdb_client import InfluxDBClient +from dotenv import load_dotenv + +load_dotenv() + +def check_scrutiny_failure_rates(): + """Check Scrutiny's own failure rate calculations.""" + url = os.getenv('INFLUXDB_URL') + token = os.getenv('INFLUXDB_TOKEN') + org = os.getenv('INFLUXDB_ORG', 'scrutiny') + bucket = os.getenv('INFLUXDB_BUCKET', 'metrics') + + client = InfluxDBClient(url=url, token=token, org=org) + query_api = client.query_api() + + # Get failure_rate fields + query = f''' + from(bucket: "{bucket}") + |> range(start: -1h) + |> filter(fn: (r) => r._measurement == "smart") + |> filter(fn: (r) => r._field =~ /failure_rate/) + |> last() + ''' + + print("=" * 80) + print("Scrutiny's Failure Rate Calculations") + print("=" * 80) + + result = query_api.query(query) + + devices = {} + for table in result: + for record in table.records: + wwn = record.values.get('device_wwn') + field = record.get_field() + value = record.get_value() + + if wwn not in devices: + devices[wwn] = {} + + # Extract attribute ID + parts = field.split('.') + if len(parts) >= 2: + attr_id = parts[1] + devices[wwn][attr_id] = value + + # Show per device + for wwn, attrs in sorted(devices.items()): + print(f"\nDevice {wwn}:") + max_rate = 0 + for attr_id, rate in sorted(attrs.items(), key=lambda x: int(x[0])): + if rate and rate > 0: + print(f" Attr {attr_id:3s}: {rate:.4f}") + max_rate = max(max_rate, rate) + + if max_rate > 0: + print(f" MAX: {max_rate:.4f} ({max_rate*100:.2f}%)") + + client.close() + +if __name__ == '__main__': + check_scrutiny_failure_rates() diff --git a/debug_smart_values.py b/debug_smart_values.py new file mode 100755 index 0000000..d149dbb --- /dev/null +++ b/debug_smart_values.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +""" +Debug script to see actual SMART attribute values from InfluxDB +""" + +import os +from influxdb_client import InfluxDBClient +from dotenv import load_dotenv + +load_dotenv() + +def debug_smart_values(): + """Fetch and display raw SMART values for debugging.""" + url = os.getenv('INFLUXDB_URL') + token = os.getenv('INFLUXDB_TOKEN') + org = os.getenv('INFLUXDB_ORG', 'scrutiny') + bucket = os.getenv('INFLUXDB_BUCKET', 'metrics') + + client = InfluxDBClient(url=url, token=token, org=org) + query_api = client.query_api() + + # Get all SMART attribute fields for one device + query = f''' + from(bucket: "{bucket}") + |> range(start: -1h) + |> filter(fn: (r) => r._measurement == "smart") + |> filter(fn: (r) => r.device_wwn == "0x5000c500c570663e") + |> filter(fn: (r) => + r._field =~ /attr\\.5\\.(raw_value|transformed_value|value)/ or + r._field =~ /attr\\.187\\.(raw_value|transformed_value|value)/ or + r._field =~ /attr\\.188\\.(raw_value|transformed_value|value)/ or + r._field =~ /attr\\.197\\.(raw_value|transformed_value|value)/ or + r._field =~ /attr\\.198\\.(raw_value|transformed_value|value)/ or + r._field =~ /attr\\.9\\.(raw_value|transformed_value|value)/ + ) + |> last() + ''' + + print("=" * 80) + print("SMART Attribute Values for /dev/sdc (ZLW0A3QJ) - Should be 81% failure") + print("=" * 80) + + result = query_api.query(query) + + values = {} + for table in result: + for record in table.records: + field = record.get_field() + value = record.get_value() + values[field] = value + + # Group by attribute + attrs = {} + for field, value in sorted(values.items()): + parts = field.split('.') + if len(parts) >= 3: + attr_id = parts[1] + field_type = parts[2] + + if attr_id not in attrs: + attrs[attr_id] = {} + attrs[attr_id][field_type] = value + + # Display + for attr_id in sorted(attrs.keys(), key=int): + print(f"\nAttribute {attr_id}:") + attr_data = attrs[attr_id] + + raw = attr_data.get('raw_value', 'N/A') + transformed = attr_data.get('transformed_value', 'N/A') + value = attr_data.get('value', 'N/A') + + print(f" raw_value: {raw}") + if isinstance(raw, int): + print(f" hex: 0x{raw:012x}") + print(f" low byte: {raw & 0xFF}") + print(f" low word (16-bit): {raw & 0xFFFF}") + print(f" low 32-bit: {raw & 0xFFFFFFFF}") + print(f" transformed_value: {transformed}") + print(f" value: {value}") + + client.close() + +if __name__ == '__main__': + debug_smart_values() diff --git a/debug_value_field.py b/debug_value_field.py new file mode 100755 index 0000000..ece991e --- /dev/null +++ b/debug_value_field.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +""" +Check the VALUE field (not raw_value) for SMART attributes +""" + +import os +from influxdb_client import InfluxDBClient +from dotenv import load_dotenv + +load_dotenv() + +url = os.getenv('INFLUXDB_URL') +token = os.getenv('INFLUXDB_TOKEN') +org = os.getenv('INFLUXDB_ORG', 'scrutiny') +bucket = os.getenv('INFLUXDB_BUCKET', 'metrics') + +client = InfluxDBClient(url=url, token=token, org=org) +query_api = client.query_api() + +# Get VALUE field (normalized 0-100) for critical attributes +query = f''' +from(bucket: "{bucket}") + |> range(start: -1h) + |> filter(fn: (r) => r._measurement == "smart") + |> filter(fn: (r) => r.device_wwn == "0x5000c500c570663e") + |> filter(fn: (r) => + r._field == "attr.5.value" or + r._field == "attr.187.value" or + r._field == "attr.188.value" or + r._field == "attr.197.value" or + r._field == "attr.198.value" or + r._field == "attr.5.raw_value" or + r._field == "attr.187.raw_value" or + r._field == "attr.188.raw_value" or + r._field == "attr.197.raw_value" or + r._field == "attr.198.raw_value" + ) + |> last() +''' + +print("=" * 80) +print("/dev/sdc (ZLW0A3QJ) - should show 81% failure") +print("Comparing InfluxDB values to smartctl output") +print("=" * 80) + +result = query_api.query(query) + +attrs = {} +for table in result: + for record in table.records: + field = record.get_field() + value = record.get_value() + + parts = field.split('.') + if len(parts) >= 3: + attr_id = parts[1] + field_type = parts[2] + + if attr_id not in attrs: + attrs[attr_id] = {} + attrs[attr_id][field_type] = value + +print("\nInfluxDB Data:") +print("-" * 80) +for attr_id in sorted(attrs.keys(), key=int): + print(f"\nAttribute {attr_id}:") + print(f" value (normalized): {attrs[attr_id].get('value', 'N/A')}") + print(f" raw_value: {attrs[attr_id].get('raw_value', 'N/A')}") + +print("\n" + "=" * 80) +print("From smartctl -A /dev/sdc:") +print("-" * 80) +print(" 5 Reallocated_Sector_Ct: VALUE=100, RAW_VALUE=0") +print("187 Reported_Uncorrect: VALUE=100, RAW_VALUE=0") +print("188 Command_Timeout: VALUE=100, RAW_VALUE=0") +print("197 Current_Pending_Sector: VALUE=100, RAW_VALUE=0") +print("198 Offline_Uncorrectable: VALUE=100, RAW_VALUE=0") + +client.close() diff --git a/device_mappings.txt b/device_mappings.txt new file mode 100644 index 0000000..28954e7 --- /dev/null +++ b/device_mappings.txt @@ -0,0 +1,10 @@ +0x5000c500744487c5 = /dev/sda [parity] +0x5000c500c455c45e = /dev/sdf +0x5000c500c46b0832 = /dev/sdi +0x5000c500c570663e = /dev/sdc [d1] +0x5000c500e584b9c8 = /dev/sdd [d5] +0x5000c500e7a6a520 = /dev/sdb [d2] +0x5000cca234c69287 = /dev/sde +0x5000cca266e83d77 = /dev/sdh [2-parity] +0x5000cca26ae062f3 = /dev/sdg [2-parity] +25243x801438 = /dev/nvme0n1 [os] diff --git a/explore_influx.py b/explore_influx.py new file mode 100755 index 0000000..4311518 --- /dev/null +++ b/explore_influx.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +""" +Explore Scrutiny's InfluxDB schema to understand how SMART data is stored. +Run this first to understand the data structure before running the main logger. +""" + +import os +from influxdb_client import InfluxDBClient +from dotenv import load_dotenv + +load_dotenv() + +def explore_influxdb(): + """Connect to InfluxDB and explore the schema.""" + url = os.getenv('INFLUXDB_URL') + token = os.getenv('INFLUXDB_TOKEN') + org = os.getenv('INFLUXDB_ORG', 'scrutiny') + bucket = os.getenv('INFLUXDB_BUCKET', 'metrics') + + print(f"Connecting to InfluxDB at {url}") + print(f"Organization: {org}") + print(f"Bucket: {bucket}\n") + + client = InfluxDBClient(url=url, token=token, org=org) + query_api = client.query_api() + + # 1. List all measurements in the bucket + print("=" * 80) + print("STEP 1: Discovering measurements in bucket") + print("=" * 80) + + query_measurements = f''' + import "influxdata/influxdb/schema" + schema.measurements(bucket: "{bucket}") + ''' + + try: + result = query_api.query(query_measurements) + measurements = [] + for table in result: + for record in table.records: + measurement = record.get_value() + measurements.append(measurement) + print(f" - {measurement}") + + if not measurements: + print(" No measurements found!") + return + + print(f"\nFound {len(measurements)} measurement(s)\n") + + # 2. For each measurement, show field keys + print("=" * 80) + print("STEP 2: Discovering fields for each measurement") + print("=" * 80) + + for measurement in measurements[:5]: # Limit to first 5 measurements + print(f"\nMeasurement: {measurement}") + query_fields = f''' + import "influxdata/influxdb/schema" + schema.measurementFieldKeys( + bucket: "{bucket}", + measurement: "{measurement}" + ) + ''' + + result = query_api.query(query_fields) + for table in result: + for record in table.records: + field = record.get_value() + print(f" Field: {field}") + + # 3. Show sample data from the first measurement + print("\n" + "=" * 80) + print("STEP 3: Sample data from first measurement") + print("=" * 80) + + first_measurement = measurements[0] + query_sample = f''' + from(bucket: "{bucket}") + |> range(start: -24h) + |> filter(fn: (r) => r._measurement == "{first_measurement}") + |> limit(n: 10) + ''' + + print(f"\nSample records from '{first_measurement}':\n") + result = query_api.query(query_sample) + for table in result: + for record in table.records: + print(f"Time: {record.get_time()}") + print(f" Measurement: {record.get_measurement()}") + print(f" Field: {record.get_field()}") + print(f" Value: {record.get_value()}") + print(f" Tags: {record.values}") + print() + + except Exception as e: + print(f"Error querying InfluxDB: {e}") + import traceback + traceback.print_exc() + + finally: + client.close() + + +if __name__ == '__main__': + if not os.getenv('INFLUXDB_URL'): + print("ERROR: Please create a .env file with your InfluxDB configuration") + print("Copy .env.example to .env and fill in your details") + exit(1) + + explore_influxdb() diff --git a/get_device_info.py b/get_device_info.py new file mode 100755 index 0000000..593e59a --- /dev/null +++ b/get_device_info.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +""" +Helper script to extract WWN to device mappings from InfluxDB. +Run this to help populate the devices table. +""" + +import os +from influxdb_client import InfluxDBClient +from dotenv import load_dotenv + +load_dotenv() + +def get_device_info(): + """Extract unique device WWNs and their latest data from InfluxDB.""" + url = os.getenv('INFLUXDB_URL') + token = os.getenv('INFLUXDB_TOKEN') + org = os.getenv('INFLUXDB_ORG', 'scrutiny') + bucket = os.getenv('INFLUXDB_BUCKET', 'metrics') + + print(f"Connecting to InfluxDB at {url}...") + client = InfluxDBClient(url=url, token=token, org=org) + query_api = client.query_api() + + # Get unique device WWNs with their latest data + query = f''' + from(bucket: "{bucket}") + |> range(start: -24h) + |> filter(fn: (r) => r._measurement == "smart") + |> filter(fn: (r) => + r._field == "temp" or + r._field == "power_on_hours" + ) + |> last() + |> group(columns: ["device_wwn", "device_protocol"]) + ''' + + print("\nDiscovered devices:\n") + print("=" * 80) + + result = query_api.query(query) + devices = {} + + for table in result: + for record in table.records: + wwn = record.values.get('device_wwn') + protocol = record.values.get('device_protocol') + + if wwn not in devices: + devices[wwn] = { + 'wwn': wwn, + 'protocol': protocol, + 'temp': None, + 'power_on_hours': None + } + + field = record.get_field() + if field == 'temp': + devices[wwn]['temp'] = record.get_value() + elif field == 'power_on_hours': + devices[wwn]['power_on_hours'] = record.get_value() + + # Print devices + for i, (wwn, info) in enumerate(sorted(devices.items()), 1): + power_on_days = int(info['power_on_hours']) // 24 if info['power_on_hours'] else 0 + + print(f"Device {i}:") + print(f" WWN: {wwn}") + print(f" Protocol: {info['protocol']}") + print(f" Temperature: {info['temp']}Ā°C") + print(f" Power-On Time: {power_on_days} days ({info['power_on_hours']} hours)") + print() + + print("=" * 80) + print("\nNow run 'snapraid smart' or 'smartctl -a /dev/sdX' on each device to get:") + print(" - Serial number") + print(" - Model name") + print(" - Capacity") + print(" - Device path (/dev/sdX)") + print("\nThen update populate_devices.sql with this information.") + + client.close() + + +if __name__ == '__main__': + if not os.getenv('INFLUXDB_URL'): + print("ERROR: Please create a .env file with your InfluxDB configuration") + exit(1) + + get_device_info() diff --git a/populate_devices.sql b/populate_devices.sql new file mode 100644 index 0000000..d641289 --- /dev/null +++ b/populate_devices.sql @@ -0,0 +1,87 @@ +-- Populate devices table with your drive metadata +-- Replace the placeholder values with actual information from your drives + +-- Device 1: WWN 0x5000c500744487c5 +INSERT INTO devices (device_wwn, device_path, serial_number, model, manufacturer, capacity_bytes, size_tb, disk_role, notes) +VALUES ( + '0x5000c500744487c5', + '/dev/sdc', -- Update with actual device path + 'ZLW0A3QJ', -- Update with actual serial + 'WDC WD120EFAX', -- Update with actual model + 'Western Digital', -- Update with manufacturer + 12000000000000, -- Update with capacity in bytes + 12.0, -- Size in TB + 'd1', -- SnapRAID role (d1, d2, parity, 2-parity, or '-' for unused) + 'Data drive 1' -- Optional notes +); + +-- Device 2: WWN 0x5000c500c455c45e +INSERT INTO devices (device_wwn, device_path, serial_number, model, manufacturer, capacity_bytes, size_tb, disk_role, notes) +VALUES ( + '0x5000c500c455c45e', + '/dev/sdb', + 'ZX22EJ3Y', + 'WDC WD200EFAX', + 'Western Digital', + 20000000000000, + 20.0, + 'd2', + 'Data drive 2' +); + +-- Device 3: WWN 0x5000c500c46b0832 +INSERT INTO devices (device_wwn, device_path, serial_number, model, manufacturer, capacity_bytes, size_tb, disk_role, notes) +VALUES ( + '0x5000c500c46b0832', + '/dev/sdd', + 'ZTN1BNEZ', + 'WDC WD120EFAX', + 'Western Digital', + 12000000000000, + 12.0, + 'd5', + 'Data drive 5' +); + +-- Device 4: WWN 0x5000c500c570663e +INSERT INTO devices (device_wwn, device_path, serial_number, model, manufacturer, capacity_bytes, size_tb, disk_role, notes) +VALUES ( + '0x5000c500c570663e', + '/dev/sda', + 'ZX20AAMV', + 'WDC WD200EFAX', + 'Western Digital', + 20000000000000, + 20.0, + 'parity', + 'Parity drive' +); + +-- Device 5: WWN 0x5000c500e584b9c8 +INSERT INTO devices (device_wwn, device_path, serial_number, model, manufacturer, capacity_bytes, size_tb, disk_role, notes) +VALUES ( + '0x5000c500e584b9c8', + '/dev/sdg', + '2TJ97M6D', + 'WDC WD100EFAX', + 'Western Digital', + 10000000000000, + 10.0, + '2-parity', + 'Second parity drive' +); + +-- Add remaining 4 devices following the same pattern +-- Template: +-- INSERT INTO devices (device_wwn, device_path, serial_number, model, manufacturer, capacity_bytes, size_tb, disk_role, notes) +-- VALUES ( +-- '0xXXXXXXXXXXXXXXXX', -- WWN from InfluxDB +-- '/dev/sdX', -- Device path +-- 'SERIAL', -- Serial number from snapraid smart or smartctl +-- 'MODEL', -- Model number +-- 'MANUFACTURER', -- Manufacturer name +-- 0, -- Capacity in bytes +-- 0.0, -- Size in TB +-- '-', -- SnapRAID role +-- '' -- Notes +-- ); diff --git a/populate_devices_final.sql b/populate_devices_final.sql new file mode 100644 index 0000000..2c59f4d --- /dev/null +++ b/populate_devices_final.sql @@ -0,0 +1,149 @@ +-- Populate devices table with actual drive data +-- Based on device_mappings.txt and snapraid smart output + +-- Device 1: /dev/sda [parity] +INSERT INTO devices (device_wwn, device_path, serial_number, model, manufacturer, capacity_bytes, size_tb, disk_role, notes) +VALUES ( + '0x5000c500744487c5', + '/dev/sda', + 'ZX20AAMV', + 'WDC WD200EFAX', + 'Western Digital', + 20000000000000, + 20.0, + 'parity', + 'Parity drive - 636 days old' +); + +-- Device 2: /dev/sdb [d2] +INSERT INTO devices (device_wwn, device_path, serial_number, model, manufacturer, capacity_bytes, size_tb, disk_role, notes) +VALUES ( + '0x5000c500e7a6a520', + '/dev/sdb', + 'ZX22EJ3Y', + 'WDC WD200EFAX', + 'Western Digital', + 20000000000000, + 20.0, + 'd2', + 'Data drive 2 - 637 days old' +); + +-- Device 3: /dev/sdc [d1] +INSERT INTO devices (device_wwn, device_path, serial_number, model, manufacturer, capacity_bytes, size_tb, disk_role, notes) +VALUES ( + '0x5000c500c570663e', + '/dev/sdc', + 'ZLW0A3QJ', + 'WDC WD120EFAX', + 'Western Digital', + 12000000000000, + 12.0, + 'd1', + 'Data drive 1 - 1968 days old - HIGH RISK 81%' +); + +-- Device 4: /dev/sdd [d5] +INSERT INTO devices (device_wwn, device_path, serial_number, model, manufacturer, capacity_bytes, size_tb, disk_role, notes) +VALUES ( + '0x5000c500e584b9c8', + '/dev/sdd', + 'ZTN1BNEZ', + 'WDC WD120EFAX', + 'Western Digital', + 12000000000000, + 12.0, + 'd5', + 'Data drive 5 - 1014 days old' +); + +-- Device 5: /dev/sde (unused) +INSERT INTO devices (device_wwn, device_path, serial_number, model, manufacturer, capacity_bytes, size_tb, disk_role, notes) +VALUES ( + '0x5000cca234c69287', + '/dev/sde', + 'YVGGG6EC', + 'WDC WD30EFRX', + 'Western Digital', + 3000000000000, + 3.0, + '-', + 'Unused drive - 1983 days old' +); + +-- Device 6: /dev/sdf (unused) +INSERT INTO devices (device_wwn, device_path, serial_number, model, manufacturer, capacity_bytes, size_tb, disk_role, notes) +VALUES ( + '0x5000c500c455c45e', + '/dev/sdf', + 'ZA1GX9F4', + 'WDC WD80EFAX', + 'Western Digital', + 8000000000000, + 8.0, + '-', + 'Unused drive - 2072 days old - HIGH RISK 84%' +); + +-- Device 7: /dev/sdg [2-parity] +INSERT INTO devices (device_wwn, device_path, serial_number, model, manufacturer, capacity_bytes, size_tb, disk_role, notes) +VALUES ( + '0x5000cca26ae062f3', + '/dev/sdg', + '2TJ97M6D', + 'WDC WD100EFAX', + 'Western Digital', + 10000000000000, + 10.0, + '2-parity', + 'Second parity drive - 2540 days old' +); + +-- Device 8: /dev/sdh [2-parity] +INSERT INTO devices (device_wwn, device_path, serial_number, model, manufacturer, capacity_bytes, size_tb, disk_role, notes) +VALUES ( + '0x5000cca266e83d77', + '/dev/sdh', + '7JJVJ65C', + 'WDC WD100EFAX', + 'Western Digital', + 10000000000000, + 10.0, + '2-parity', + 'Second parity drive - 2354 days old - 1 error reported' +); + +-- Device 9: /dev/sdi (unused) +INSERT INTO devices (device_wwn, device_path, serial_number, model, manufacturer, capacity_bytes, size_tb, disk_role, notes) +VALUES ( + '0x5000c500c46b0832', + '/dev/sdi', + 'ZA1GW0XM', + 'WDC WD80EFAX', + 'Western Digital', + 8000000000000, + 8.0, + '-', + 'Unused drive - 2074 days old - HIGH RISK 84%' +); + +-- Device 10: /dev/nvme0n1 [OS drive] - Note: WWN format different for NVMe +-- NVMe devices may not report WWN in the same format, skipping for now +-- You can add this manually if needed after checking actual WWN from InfluxDB +-- INSERT INTO devices (device_wwn, device_path, serial_number, model, manufacturer, capacity_bytes, size_tb, disk_role, notes) +-- VALUES ( +-- '25243x801438', -- May need different format +-- '/dev/nvme0n1', +-- '25243X801438', +-- 'NVMe SSD', +-- 'Unknown', +-- NULL, +-- NULL, +-- 'os', +-- 'Operating system drive' +-- ); + +-- Verify the data +SELECT device_path, serial_number, size_tb, disk_role, notes +FROM devices +ORDER BY device_path; diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..13eac51 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +influxdb-client>=1.38.0 +psycopg[binary]>=3.1.0 +python-dotenv>=1.0.0 +requests>=2.31.0 diff --git a/schema.sql b/schema.sql new file mode 100644 index 0000000..a67fe5c --- /dev/null +++ b/schema.sql @@ -0,0 +1,58 @@ +-- PostgreSQL schema for SMART data logging + +-- Main table for SMART metrics snapshots +CREATE TABLE IF NOT EXISTS smart_metrics ( + id BIGSERIAL PRIMARY KEY, + timestamp TIMESTAMPTZ NOT NULL DEFAULT NOW(), + device_path VARCHAR(255) NOT NULL, + serial_number VARCHAR(255) NOT NULL, + model VARCHAR(255), + disk_role VARCHAR(50), -- e.g., 'd1', 'd2', 'parity', '2-parity', or '-' for unused + + -- Basic metrics (from SnapRAID smart output) + temperature_celsius INTEGER, + power_on_days INTEGER, + error_count INTEGER, + size_tb DECIMAL(10, 2), + + -- Calculated failure probability + failure_probability_pct DECIMAL(5, 2), -- 0.00 to 100.00 + + -- Raw SMART attributes (key-value pairs as JSONB) + smart_attributes JSONB, + + -- Indexes for efficient querying + CONSTRAINT unique_device_timestamp UNIQUE (device_path, timestamp) +); + +-- Index on serial number for device history queries +CREATE INDEX IF NOT EXISTS idx_smart_metrics_serial ON smart_metrics(serial_number); + +-- Index on timestamp for time-range queries +CREATE INDEX IF NOT EXISTS idx_smart_metrics_timestamp ON smart_metrics(timestamp DESC); + +-- Index on device_path for per-device queries +CREATE INDEX IF NOT EXISTS idx_smart_metrics_device ON smart_metrics(device_path); + +-- Index on JSONB attributes for querying specific SMART values +CREATE INDEX IF NOT EXISTS idx_smart_attributes_gin ON smart_metrics USING gin (smart_attributes); + +-- View for latest metrics per device +CREATE OR REPLACE VIEW smart_latest AS +SELECT DISTINCT ON (device_path) + * +FROM smart_metrics +ORDER BY device_path, timestamp DESC; + +-- View for high-risk devices (failure probability > 50%) +CREATE OR REPLACE VIEW smart_high_risk AS +SELECT * +FROM smart_latest +WHERE failure_probability_pct > 50.0 +ORDER BY failure_probability_pct DESC; + +COMMENT ON TABLE smart_metrics IS 'Historical SMART metrics collected from Scrutiny InfluxDB'; +COMMENT ON COLUMN smart_metrics.failure_probability_pct IS 'BackBlaze-based 1-year failure probability (0-100%)'; +COMMENT ON COLUMN smart_metrics.smart_attributes IS 'Raw SMART attributes as JSON (attribute_id -> value)'; +COMMENT ON VIEW smart_latest IS 'Latest SMART metrics for each device'; +COMMENT ON VIEW smart_high_risk IS 'Devices with >50% annual failure probability'; diff --git a/schema_migration_add_v13.sql b/schema_migration_add_v13.sql new file mode 100644 index 0000000..5aa550b --- /dev/null +++ b/schema_migration_add_v13.sql @@ -0,0 +1,72 @@ +-- Migration: Add v13.0 failure probability column +-- Adds a second failure probability column for the modern algorithm (without attr 193) + +-- Add new column for v13.0+ algorithm (without attribute 193) +ALTER TABLE smart_metrics +ADD COLUMN IF NOT EXISTS failure_probability_v13_pct DECIMAL(5, 2); + +-- Rename existing column to clarify it includes attribute 193 (pre-v13.0) +ALTER TABLE smart_metrics +RENAME COLUMN failure_probability_pct TO failure_probability_v12_pct; + +-- Update views to include both columns +DROP VIEW IF EXISTS smart_latest CASCADE; +CREATE VIEW smart_latest AS +SELECT + m.id, + m.timestamp, + m.device_wwn, + d.device_path, + d.serial_number, + d.model, + d.manufacturer, + d.capacity_bytes, + d.size_tb, + d.disk_role, + m.temperature_celsius, + m.power_on_days, + m.error_count, + m.failure_probability_v12_pct, -- With attr 193 (matches your current SnapRAID) + m.failure_probability_v13_pct, -- Without attr 193 (modern SnapRAID v13.0+) + m.smart_attributes +FROM smart_metrics m +INNER JOIN devices d ON m.device_wwn = d.device_wwn +WHERE m.timestamp = ( + SELECT MAX(m2.timestamp) + FROM smart_metrics m2 + WHERE m2.device_wwn = m.device_wwn +); + +DROP VIEW IF EXISTS smart_high_risk; +CREATE VIEW smart_high_risk AS +SELECT * +FROM smart_latest +WHERE failure_probability_v12_pct > 50 -- Using v12 since that matches your SnapRAID +ORDER BY failure_probability_v12_pct DESC; + +DROP VIEW IF EXISTS smart_summary; +CREATE VIEW smart_summary AS +SELECT + device_path, + serial_number, + temperature_celsius, + power_on_days, + error_count, + failure_probability_v12_pct, -- With attr 193 + failure_probability_v13_pct, -- Without attr 193 + size_tb, + disk_role +FROM smart_latest +ORDER BY + CASE disk_role + WHEN 'parity' THEN 1 + WHEN '2-parity' THEN 2 + ELSE 3 + END, + device_path; + +COMMENT ON COLUMN smart_metrics.failure_probability_v12_pct IS + 'Failure probability using SnapRAID 50%) +CREATE OR REPLACE VIEW smart_high_risk AS +SELECT * +FROM smart_latest +WHERE failure_probability_pct > 50.0 +ORDER BY failure_probability_pct DESC; + +-- View for summary (like snapraid smart output) +CREATE OR REPLACE VIEW smart_summary AS +SELECT + device_path, + serial_number, + temperature_celsius as temp_c, + power_on_days, + error_count, + failure_probability_pct as fp_pct, + size_tb, + disk_role, + model, + timestamp as last_updated +FROM smart_latest +ORDER BY + CASE disk_role + WHEN 'parity' THEN 1 + WHEN '2-parity' THEN 2 + ELSE 3 + END, + device_path; + +-- Comments +COMMENT ON TABLE devices IS 'Device metadata (WWN, serial, model, etc.) - manually populated'; +COMMENT ON TABLE smart_metrics IS 'Historical SMART metrics collected from Scrutiny InfluxDB'; +COMMENT ON COLUMN smart_metrics.failure_probability_pct IS 'BackBlaze-based 1-year failure probability (0-100%)'; +COMMENT ON COLUMN smart_metrics.smart_attributes IS 'Raw SMART attributes as JSON (attribute_id -> value)'; +COMMENT ON VIEW smart_latest IS 'Latest SMART metrics for each device with metadata'; +COMMENT ON VIEW smart_high_risk IS 'Devices with >50% annual failure probability'; +COMMENT ON VIEW smart_summary IS 'Summary view matching snapraid smart output format'; diff --git a/setup_database.sh b/setup_database.sh new file mode 100755 index 0000000..595b017 --- /dev/null +++ b/setup_database.sh @@ -0,0 +1,87 @@ +#!/bin/bash +# Setup script for PostgreSQL database and user + +set -e # Exit on error + +# Configuration +PG_HOST="${1:-localhost}" +PG_ADMIN_USER="postgres" +PG_ADMIN_PASS="Lkxdjfhl1d" + +DB_NAME="smart_monitoring" +DB_USER="smart_logger" +DB_PASS="${2:-change_me_123}" # Pass as second argument or use default + +echo "PostgreSQL Setup for Smart Logger" +echo "==================================" +echo "Host: $PG_HOST" +echo "Database: $DB_NAME" +echo "User: $DB_USER" +echo "" + +# Check if psql is installed +if ! command -v psql &> /dev/null; then + echo "ERROR: psql is not installed" + echo "Install with: sudo pacman -S postgresql" + exit 1 +fi + +echo "Step 1: Creating database..." +PGPASSWORD="$PG_ADMIN_PASS" psql -h "$PG_HOST" -U "$PG_ADMIN_USER" -d postgres < List[Dict]: + """ + Fetch latest SMART data from InfluxDB. + + NOTE: This query needs to be customized based on your Scrutiny schema. + Run explore_influx.py first to understand the measurement and field names. + + Returns: + List of device data dictionaries + """ + query_api = self.influx_client.query_api() + + # TODO: Customize this query based on your Scrutiny InfluxDB schema + # This is a template - you'll need to adjust field names and tags + query = f''' + from(bucket: "{self.influx_bucket}") + |> range(start: -5m) + |> filter(fn: (r) => r._measurement == "smart") + |> last() + |> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value") + ''' + + print("Querying InfluxDB for SMART data...") + result = query_api.query(query) + + devices = [] + for table in result: + for record in table.records: + # Extract device information + # TODO: Adjust these field names based on your schema + device_data = { + 'timestamp': record.get_time(), + 'device_path': record.values.get('device', 'unknown'), + 'serial_number': record.values.get('serial', 'unknown'), + 'model': record.values.get('model', 'unknown'), + 'temperature': record.values.get('temp', None), + 'power_on_hours': record.values.get('power_on_hours', None), + 'smart_attributes': {} + } + + # Collect SMART attributes (IDs 5, 187, 188, 197, 198) + # TODO: Map your InfluxDB field names to SMART attribute IDs + for attr_id in [5, 187, 188, 197, 198]: + field_name = f'attr_{attr_id}' # Adjust based on your schema + if field_name in record.values: + device_data['smart_attributes'][attr_id] = record.values[field_name] + + devices.append(device_data) + + print(f"āœ“ Found {len(devices)} device(s)") + return devices + + def process_device_data(self, devices: List[Dict]) -> List[Dict]: + """ + Process device data and calculate failure probabilities. + + Args: + devices: Raw device data from InfluxDB + + Returns: + Processed device data with calculated AFR + """ + processed = [] + + for device in devices: + # Determine if this is a Seagate disk + model = device.get('model', '').lower() + is_seagate = 'seagate' in model + + # Calculate failure probability + smart_attrs = device.get('smart_attributes', {}) + afr = calculate_afr(smart_attrs, is_seagate=is_seagate) + failure_probability_pct = afr * 100 # Convert to percentage + + # Calculate power-on days + power_on_hours = device.get('power_on_hours') + power_on_days = power_on_hours // 24 if power_on_hours else None + + # Get error count (SMART attribute 199 or similar) + error_count = smart_attrs.get(199, 0) # TODO: Adjust based on your needs + + processed_device = { + 'timestamp': device.get('timestamp', datetime.now(timezone.utc)), + 'device_path': device.get('device_path'), + 'serial_number': device.get('serial_number'), + 'model': device.get('model'), + 'disk_role': None, # TODO: Map device to snapraid role if needed + 'temperature_celsius': device.get('temperature'), + 'power_on_days': power_on_days, + 'error_count': error_count, + 'size_tb': None, # TODO: Get disk size if available + 'failure_probability_pct': round(failure_probability_pct, 2), + 'smart_attributes': json.dumps(smart_attrs) + } + + processed.append(processed_device) + + # Print summary + print(f"\nDevice: {processed_device['device_path']}") + print(f" Serial: {processed_device['serial_number']}") + print(f" Model: {processed_device['model']}") + print(f" Temp: {processed_device['temperature_celsius']}Ā°C") + print(f" Power-On: {processed_device['power_on_days']} days") + print(f" Failure Probability: {processed_device['failure_probability_pct']:.2f}%") + + return processed + + def save_to_postgres(self, devices: List[Dict]): + """ + Save processed device data to PostgreSQL. + + Args: + devices: Processed device data + """ + if not devices: + print("No devices to save") + return + + print(f"\nSaving {len(devices)} device record(s) to PostgreSQL...") + + insert_query = ''' + INSERT INTO smart_metrics ( + timestamp, device_path, serial_number, model, disk_role, + temperature_celsius, power_on_days, error_count, size_tb, + failure_probability_pct, smart_attributes + ) VALUES ( + %(timestamp)s, %(device_path)s, %(serial_number)s, %(model)s, %(disk_role)s, + %(temperature_celsius)s, %(power_on_days)s, %(error_count)s, %(size_tb)s, + %(failure_probability_pct)s, %(smart_attributes)s::jsonb + ) + ON CONFLICT (device_path, timestamp) DO UPDATE SET + temperature_celsius = EXCLUDED.temperature_celsius, + power_on_days = EXCLUDED.power_on_days, + error_count = EXCLUDED.error_count, + failure_probability_pct = EXCLUDED.failure_probability_pct, + smart_attributes = EXCLUDED.smart_attributes + ''' + + with self.pg_conn.cursor() as cur: + for device in devices: + cur.execute(insert_query, device) + + self.pg_conn.commit() + print("āœ“ Data saved to PostgreSQL") + + def run(self): + """Main execution flow.""" + try: + self.connect_influxdb() + self.connect_postgres() + + devices = self.fetch_smart_data() + processed = self.process_device_data(devices) + self.save_to_postgres(processed) + + print("\n" + "=" * 80) + print("Summary:") + print("=" * 80) + for device in processed: + fp = device['failure_probability_pct'] + status = "šŸ”“ HIGH RISK" if fp > 50 else "šŸŸ” MEDIUM" if fp > 20 else "šŸŸ¢ OK" + print(f"{status} {device['device_path']}: {fp:.2f}% failure probability") + + except Exception as e: + print(f"ERROR: {e}") + import traceback + traceback.print_exc() + sys.exit(1) + + finally: + if self.influx_client: + self.influx_client.close() + if self.pg_conn: + self.pg_conn.close() + + +def main(): + # Validate environment variables + required_vars = ['INFLUXDB_URL', 'INFLUXDB_TOKEN', 'POSTGRES_PASSWORD'] + missing = [var for var in required_vars if not os.getenv(var)] + + if missing: + print(f"ERROR: Missing required environment variables: {', '.join(missing)}") + print("Please create a .env file based on .env.example") + sys.exit(1) + + logger = SmartLogger() + logger.run() + + +if __name__ == '__main__': + main() diff --git a/smart_logger_v2.py b/smart_logger_v2.py new file mode 100755 index 0000000..3a420b5 --- /dev/null +++ b/smart_logger_v2.py @@ -0,0 +1,335 @@ +#!/usr/bin/env python3 +""" +SMART Logger V2 - Optimized for Scrutiny's actual InfluxDB schema. + +Fetches SMART data from Scrutiny's InfluxDB and device metadata from Scrutiny's API, +calculates BackBlaze failure probability, and logs to PostgreSQL. +""" + +import os +import sys +from datetime import datetime, timezone, timedelta +from typing import Dict, List, Optional +import json +import requests + +from influxdb_client import InfluxDBClient +import psycopg +from psycopg.rows import dict_row +from dotenv import load_dotenv + +from backblaze_tables import calculate_afr + +load_dotenv() + + +class SmartLogger: + def __init__(self): + # InfluxDB configuration + self.influx_url = os.getenv('INFLUXDB_URL') + self.influx_token = os.getenv('INFLUXDB_TOKEN') + self.influx_org = os.getenv('INFLUXDB_ORG', 'scrutiny') + self.influx_bucket = os.getenv('INFLUXDB_BUCKET', 'metrics') + + # Scrutiny API configuration + self.scrutiny_api_url = os.getenv('SCRUTINY_API_URL', + self.influx_url.replace(':8086', ':8080')) + + # PostgreSQL configuration + self.pg_host = os.getenv('POSTGRES_HOST', 'localhost') + self.pg_port = int(os.getenv('POSTGRES_PORT', '5432')) + self.pg_db = os.getenv('POSTGRES_DB', 'smart_monitoring') + self.pg_user = os.getenv('POSTGRES_USER', 'postgres') + self.pg_password = os.getenv('POSTGRES_PASSWORD') + + self.influx_client = None + self.pg_conn = None + self.device_metadata = {} # WWN -> device info mapping + + def connect_influxdb(self): + """Connect to InfluxDB.""" + print(f"Connecting to InfluxDB at {self.influx_url}...") + self.influx_client = InfluxDBClient( + url=self.influx_url, + token=self.influx_token, + org=self.influx_org + ) + print("āœ“ Connected to InfluxDB") + + def connect_postgres(self): + """Connect to PostgreSQL.""" + print(f"Connecting to PostgreSQL at {self.pg_host}:{self.pg_port}...") + self.pg_conn = psycopg.connect( + host=self.pg_host, + port=self.pg_port, + dbname=self.pg_db, + user=self.pg_user, + password=self.pg_password, + row_factory=dict_row + ) + print("āœ“ Connected to PostgreSQL") + + def fetch_device_metadata(self): + """ + Fetch device metadata from Scrutiny API. + This gets serial, model, device path, etc. + """ + print("Fetching device metadata from Scrutiny API...") + try: + response = requests.get(f"{self.scrutiny_api_url}/api/summary") + response.raise_for_status() + data = response.json() + + # Parse the summary response + if 'data' in data and 'summary' in data['data']: + for wwn, device_info in data['data']['summary'].items(): + device = device_info.get('device', {}) + self.device_metadata[wwn] = { + 'serial_number': device.get('serial_number', 'unknown'), + 'model': device.get('model_name', 'unknown'), + 'device_name': device.get('device_name', 'unknown'), + 'device_type': device.get('device_type', 'unknown'), + 'manufacturer': device.get('manufacturer', 'unknown'), + 'capacity': device.get('capacity', 0) + } + + print(f"āœ“ Found metadata for {len(self.device_metadata)} device(s)") + + except Exception as e: + print(f"āš  Warning: Could not fetch device metadata from API: {e}") + print(" Will use WWN as identifier") + + def fetch_smart_data_from_influx(self) -> Dict[str, Dict]: + """ + Fetch latest SMART data from InfluxDB for all devices. + + Returns: + Dict mapping WWN to device data + """ + query_api = self.influx_client.query_api() + + # Query to get latest data for each device + query = f''' + from(bucket: "{self.influx_bucket}") + |> range(start: -1h) + |> filter(fn: (r) => r._measurement == "smart") + |> filter(fn: (r) => + r._field == "temp" or + r._field == "power_on_hours" or + r._field == "attr.5.raw_value" or + r._field == "attr.187.raw_value" or + r._field == "attr.188.raw_value" or + r._field == "attr.197.raw_value" or + r._field == "attr.198.raw_value" or + r._field == "attr.9.raw_value" + ) + |> last() + ''' + + print("Querying InfluxDB for SMART data...") + result = query_api.query(query) + + devices_data = {} + + for table in result: + for record in table.records: + wwn = record.values.get('device_wwn') + if not wwn: + continue + + if wwn not in devices_data: + devices_data[wwn] = { + 'timestamp': record.get_time(), + 'wwn': wwn, + 'protocol': record.values.get('device_protocol'), + 'smart_attributes': {}, + 'temp': None, + 'power_on_hours': None + } + + field = record.get_field() + value = record.get_value() + + # Map fields to device data + if field == 'temp': + devices_data[wwn]['temp'] = int(value) if value else None + elif field == 'power_on_hours': + devices_data[wwn]['power_on_hours'] = int(value) if value else None + elif field.startswith('attr.'): + # Extract attribute ID from field name like "attr.5.raw_value" + parts = field.split('.') + if len(parts) >= 2: + attr_id = int(parts[1]) + # We want raw_value for calculations + if parts[-1] == 'raw_value': + # For most attributes, we want the raw value modulo 2^32 + # (InfluxDB stores as int64, but SMART is typically 48-bit) + raw_val = int(value) if value else 0 + # Mask to handle potential large values + devices_data[wwn]['smart_attributes'][attr_id] = raw_val & 0xFFFF + + print(f"āœ“ Found data for {len(devices_data)} device(s)") + return devices_data + + def process_device_data(self, devices_data: Dict[str, Dict]) -> List[Dict]: + """ + Process device data and calculate failure probabilities. + + Args: + devices_data: Dict mapping WWN to device data from InfluxDB + + Returns: + Processed device data with calculated AFR + """ + processed = [] + + for wwn, data in devices_data.items(): + # Get device metadata + metadata = self.device_metadata.get(wwn, {}) + + # Determine if this is a Seagate disk + model = metadata.get('model', '').lower() + manufacturer = metadata.get('manufacturer', '').lower() + is_seagate = 'seagate' in model or 'seagate' in manufacturer + + # Calculate failure probability + smart_attrs = data.get('smart_attributes', {}) + afr = calculate_afr(smart_attrs, is_seagate=is_seagate) + failure_probability_pct = afr * 100 # Convert to percentage + + # Calculate power-on days + power_on_hours = data.get('power_on_hours') + power_on_days = power_on_hours // 24 if power_on_hours else None + + # Calculate disk size in TB + capacity_bytes = metadata.get('capacity', 0) + size_tb = round(capacity_bytes / (1000**4), 1) if capacity_bytes else None + + # Get device path/name + device_name = metadata.get('device_name', wwn) + + processed_device = { + 'timestamp': data.get('timestamp', datetime.now(timezone.utc)), + 'device_path': device_name, + 'serial_number': metadata.get('serial_number', wwn), + 'model': metadata.get('model', 'unknown'), + 'disk_role': None, # TODO: Map to snapraid role if needed + 'temperature_celsius': data.get('temp'), + 'power_on_days': power_on_days, + 'error_count': smart_attrs.get(199, smart_attrs.get(187, 0)), + 'size_tb': size_tb, + 'failure_probability_pct': round(failure_probability_pct, 2), + 'smart_attributes': json.dumps(smart_attrs) + } + + processed.append(processed_device) + + # Print summary + print(f"\nDevice: {processed_device['device_path']}") + print(f" Serial: {processed_device['serial_number']}") + print(f" Model: {processed_device['model']}") + print(f" Temp: {processed_device['temperature_celsius']}Ā°C") + print(f" Power-On: {processed_device['power_on_days']} days") + print(f" Size: {processed_device['size_tb']} TB") + print(f" SMART Attrs: {list(smart_attrs.keys())}") + print(f" Failure Probability: {processed_device['failure_probability_pct']:.2f}%") + + return processed + + def save_to_postgres(self, devices: List[Dict]): + """ + Save processed device data to PostgreSQL. + + Args: + devices: Processed device data + """ + if not devices: + print("No devices to save") + return + + print(f"\nSaving {len(devices)} device record(s) to PostgreSQL...") + + insert_query = ''' + INSERT INTO smart_metrics ( + timestamp, device_path, serial_number, model, disk_role, + temperature_celsius, power_on_days, error_count, size_tb, + failure_probability_pct, smart_attributes + ) VALUES ( + %(timestamp)s, %(device_path)s, %(serial_number)s, %(model)s, %(disk_role)s, + %(temperature_celsius)s, %(power_on_days)s, %(error_count)s, %(size_tb)s, + %(failure_probability_pct)s, %(smart_attributes)s::jsonb + ) + ON CONFLICT (device_path, timestamp) DO UPDATE SET + temperature_celsius = EXCLUDED.temperature_celsius, + power_on_days = EXCLUDED.power_on_days, + error_count = EXCLUDED.error_count, + failure_probability_pct = EXCLUDED.failure_probability_pct, + smart_attributes = EXCLUDED.smart_attributes + ''' + + with self.pg_conn.cursor() as cur: + for device in devices: + cur.execute(insert_query, device) + + self.pg_conn.commit() + print("āœ“ Data saved to PostgreSQL") + + def run(self): + """Main execution flow.""" + try: + self.connect_influxdb() + self.connect_postgres() + + # Fetch device metadata from Scrutiny API + self.fetch_device_metadata() + + # Fetch SMART data from InfluxDB + devices_data = self.fetch_smart_data_from_influx() + + # Process and calculate failure probabilities + processed = self.process_device_data(devices_data) + + # Save to PostgreSQL + self.save_to_postgres(processed) + + # Print summary + print("\n" + "=" * 80) + print("Summary:") + print("=" * 80) + for device in sorted(processed, key=lambda x: x['failure_probability_pct'], reverse=True): + fp = device['failure_probability_pct'] + status = "šŸ”“ HIGH RISK" if fp > 50 else "šŸŸ” MEDIUM" if fp > 20 else "šŸŸ¢ OK" + print(f"{status} {device['device_path']:20s} {fp:5.2f}% | " + f"Temp: {device['temperature_celsius']:2d}Ā°C | " + f"Age: {device['power_on_days']:5d} days | " + f"Size: {device['size_tb']:4.1f} TB") + + except Exception as e: + print(f"ERROR: {e}") + import traceback + traceback.print_exc() + sys.exit(1) + + finally: + if self.influx_client: + self.influx_client.close() + if self.pg_conn: + self.pg_conn.close() + + +def main(): + # Validate environment variables + required_vars = ['INFLUXDB_URL', 'INFLUXDB_TOKEN', 'POSTGRES_PASSWORD'] + missing = [var for var in required_vars if not os.getenv(var)] + + if missing: + print(f"ERROR: Missing required environment variables: {', '.join(missing)}") + print("Please create a .env file based on .env.example") + sys.exit(1) + + logger = SmartLogger() + logger.run() + + +if __name__ == '__main__': + main() diff --git a/smart_logger_v3.py b/smart_logger_v3.py new file mode 100755 index 0000000..6779e7f --- /dev/null +++ b/smart_logger_v3.py @@ -0,0 +1,387 @@ +#!/usr/bin/env python3 +""" +SMART Logger V3 - Uses PostgreSQL devices table for metadata. + +Fetches SMART data from Scrutiny's InfluxDB, looks up device metadata from +PostgreSQL devices table, calculates BackBlaze failure probability, and logs +the enhanced data to PostgreSQL. +""" + +import os +import sys +from datetime import datetime, timezone +from typing import Dict, List, Optional +import json + +from influxdb_client import InfluxDBClient +import psycopg +from psycopg.rows import dict_row +from dotenv import load_dotenv + +from backblaze_tables import calculate_afr, afr_to_failure_probability + +load_dotenv() + + +class SmartLogger: + def __init__(self): + # InfluxDB configuration + self.influx_url = os.getenv('INFLUXDB_URL') + self.influx_token = os.getenv('INFLUXDB_TOKEN') + self.influx_org = os.getenv('INFLUXDB_ORG', 'scrutiny') + self.influx_bucket = os.getenv('INFLUXDB_BUCKET', 'metrics') + + # PostgreSQL configuration + self.pg_host = os.getenv('POSTGRES_HOST', 'localhost') + self.pg_port = int(os.getenv('POSTGRES_PORT', '5432')) + self.pg_db = os.getenv('POSTGRES_DB', 'smart_monitoring') + self.pg_user = os.getenv('POSTGRES_USER', 'postgres') + self.pg_password = os.getenv('POSTGRES_PASSWORD') + + self.influx_client = None + self.pg_conn = None + self.device_metadata = {} # WWN -> device info mapping + + def connect_influxdb(self): + """Connect to InfluxDB.""" + print(f"Connecting to InfluxDB at {self.influx_url}...") + self.influx_client = InfluxDBClient( + url=self.influx_url, + token=self.influx_token, + org=self.influx_org + ) + print("āœ“ Connected to InfluxDB") + + def connect_postgres(self): + """Connect to PostgreSQL.""" + print(f"Connecting to PostgreSQL at {self.pg_host}:{self.pg_port}...") + self.pg_conn = psycopg.connect( + host=self.pg_host, + port=self.pg_port, + dbname=self.pg_db, + user=self.pg_user, + password=self.pg_password, + row_factory=dict_row + ) + print("āœ“ Connected to PostgreSQL") + + def load_device_metadata(self): + """Load device metadata from PostgreSQL devices table.""" + print("Loading device metadata from database...") + + query = """ + SELECT device_wwn, device_path, serial_number, model, manufacturer, + capacity_bytes, size_tb, disk_role, notes + FROM devices + """ + + with self.pg_conn.cursor() as cur: + cur.execute(query) + for row in cur.fetchall(): + wwn = row['device_wwn'] + self.device_metadata[wwn] = dict(row) + + print(f"āœ“ Loaded metadata for {len(self.device_metadata)} device(s)") + + if not self.device_metadata: + print("\nāš  WARNING: No devices found in the 'devices' table!") + print(" Please populate the devices table using populate_devices.sql") + print(" Run: psql -h -U -d -f populate_devices.sql\n") + + def fetch_smart_data_from_influx(self) -> Dict[str, Dict]: + """ + Fetch latest SMART data from InfluxDB for all devices. + + Returns: + Dict mapping WWN to device data + """ + query_api = self.influx_client.query_api() + + # Query to get latest data for each device + # Include attribute 193 for SnapRAID pre-v13.0 compatibility + query = f''' + from(bucket: "{self.influx_bucket}") + |> range(start: -1h) + |> filter(fn: (r) => r._measurement == "smart") + |> filter(fn: (r) => + r._field == "temp" or + r._field == "power_on_hours" or + r._field =~ /failure_rate$/ or + r._field == "attr.5.raw_value" or + r._field == "attr.187.raw_value" or + r._field == "attr.188.raw_value" or + r._field == "attr.193.raw_value" or + r._field == "attr.197.raw_value" or + r._field == "attr.198.raw_value" or + r._field == "attr.9.raw_value" + ) + |> last() + ''' + + print("Querying InfluxDB for SMART data...") + result = query_api.query(query) + + devices_data = {} + + for table in result: + for record in table.records: + wwn = record.values.get('device_wwn') + if not wwn: + continue + + if wwn not in devices_data: + devices_data[wwn] = { + 'timestamp': record.get_time(), + 'wwn': wwn, + 'protocol': record.values.get('device_protocol'), + 'smart_attributes': {}, + 'failure_rates': {}, # Scrutiny's pre-calculated rates + 'temp': None, + 'power_on_hours': None + } + + field = record.get_field() + value = record.get_value() + + # Map fields to device data + if field == 'temp': + devices_data[wwn]['temp'] = int(value) if value else None + elif field == 'power_on_hours': + devices_data[wwn]['power_on_hours'] = int(value) if value else None + elif field.startswith('attr.'): + # Extract attribute ID from field name like "attr.5.raw_value" + parts = field.split('.') + if len(parts) >= 2: + # Try to parse as integer, skip if not numeric (e.g., NVMe attributes) + try: + attr_id = int(parts[1]) + except ValueError: + continue # Skip non-numeric attribute IDs (NVMe, etc.) + + # Collect raw_value for logging + if parts[-1] == 'raw_value': + raw_val = int(value) if value else 0 + # Apply correct bit mask per attribute (like SnapRAID does) + # 16-bit mask for: 187 (Uncorrectable), 188 (Timeout) + # 32-bit mask for: 5 (Reallocated), 193 (Load Cycle), 197 (Pending), 198 (Offline) + if attr_id in [187, 188]: + devices_data[wwn]['smart_attributes'][attr_id] = raw_val & 0xFFFF + else: + devices_data[wwn]['smart_attributes'][attr_id] = raw_val & 0xFFFFFFFF + # Collect Scrutiny's failure_rate for each attribute + elif parts[-1] == 'failure_rate': + devices_data[wwn]['failure_rates'][attr_id] = float(value) if value else 0.0 + + print(f"āœ“ Found data for {len(devices_data)} device(s)") + return devices_data + + def process_device_data(self, devices_data: Dict[str, Dict]) -> List[Dict]: + """ + Process device data and calculate failure probabilities. + + Args: + devices_data: Dict mapping WWN to device data from InfluxDB + + Returns: + Processed device data with calculated AFR + """ + processed = [] + + for wwn, data in devices_data.items(): + # Get device metadata from database + metadata = self.device_metadata.get(wwn, {}) + + if not metadata: + print(f"\nāš  Warning: No metadata found for device {wwn}") + print(f" Add this device to the devices table using populate_devices.sql") + continue + + # Get SMART attributes + smart_attrs = data.get('smart_attributes', {}) + + # Determine if Seagate (for attribute 188 handling) + model = metadata.get('model', '').lower() + manufacturer = metadata.get('manufacturer', '').lower() + is_seagate = 'seagate' in model or 'seagate' in manufacturer + + # Calculate failure probability BOTH ways for trending + # v12 (with attr 193) - matches your current SnapRAID + afr_v12 = calculate_afr(smart_attrs, is_seagate=is_seagate, include_193=True) + prob_v12 = afr_to_failure_probability(afr_v12) + failure_probability_v12_pct = prob_v12 * 100 + + # v13 (without attr 193) - modern SnapRAID v13.0+ + afr_v13 = calculate_afr(smart_attrs, is_seagate=is_seagate, include_193=False) + prob_v13 = afr_to_failure_probability(afr_v13) + failure_probability_v13_pct = prob_v13 * 100 + + # Calculate power-on days + power_on_hours = data.get('power_on_hours') + power_on_days = power_on_hours // 24 if power_on_hours else None + + processed_device = { + 'timestamp': data.get('timestamp', datetime.now(timezone.utc)), + 'device_wwn': wwn, + 'temperature_celsius': data.get('temp'), + 'power_on_days': power_on_days, + 'error_count': smart_attrs.get(199, smart_attrs.get(187, 0)), + 'failure_probability_v12_pct': round(failure_probability_v12_pct, 2), + 'failure_probability_v13_pct': round(failure_probability_v13_pct, 2), + 'smart_attributes': json.dumps(smart_attrs) + } + + processed.append(processed_device) + + # Print summary with metadata + print(f"\n{metadata.get('device_path', wwn)}") + print(f" WWN: {wwn}") + print(f" Serial: {metadata.get('serial_number', 'unknown')}") + print(f" Model: {metadata.get('model', 'unknown')}") + print(f" Role: {metadata.get('disk_role', '-')}") + print(f" Temp: {processed_device['temperature_celsius']}Ā°C") + print(f" Power-On: {processed_device['power_on_days']} days") + print(f" Size: {metadata.get('size_tb', '?')} TB") + print(f" SMART Attrs: {list(smart_attrs.keys())}") + print(f" Failure Probability (v12 w/193): {processed_device['failure_probability_v12_pct']:.2f}%") + print(f" Failure Probability (v13 no193): {processed_device['failure_probability_v13_pct']:.2f}%") + + return processed + + def save_to_postgres(self, devices: List[Dict]): + """ + Save processed device data to PostgreSQL. + + Args: + devices: Processed device data + """ + if not devices: + print("\nNo devices to save") + return + + print(f"\nSaving {len(devices)} device record(s) to PostgreSQL...") + + insert_query = ''' + INSERT INTO smart_metrics ( + timestamp, device_wwn, + temperature_celsius, power_on_days, error_count, + failure_probability_v12_pct, failure_probability_v13_pct, smart_attributes + ) VALUES ( + %(timestamp)s, %(device_wwn)s, + %(temperature_celsius)s, %(power_on_days)s, %(error_count)s, + %(failure_probability_v12_pct)s, %(failure_probability_v13_pct)s, %(smart_attributes)s::jsonb + ) + ON CONFLICT (device_wwn, timestamp) DO UPDATE SET + temperature_celsius = EXCLUDED.temperature_celsius, + power_on_days = EXCLUDED.power_on_days, + error_count = EXCLUDED.error_count, + failure_probability_v12_pct = EXCLUDED.failure_probability_v12_pct, + failure_probability_v13_pct = EXCLUDED.failure_probability_v13_pct, + smart_attributes = EXCLUDED.smart_attributes + ''' + + with self.pg_conn.cursor() as cur: + for device in devices: + cur.execute(insert_query, device) + + self.pg_conn.commit() + print("āœ“ Data saved to PostgreSQL") + + def print_summary(self): + """Print a summary with both v12 and v13 failure calculations.""" + print("\n" + "=" * 95) + print("SMART Summary (v12=with attr 193, v13=without attr 193)") + print("=" * 95) + print(f"{'Temp':>5} {'Power':>6} {'Error':>7} {'v12':>4} {'v13':>4} {'Size':>5}") + print(f"{'C':>5} {'OnDays':>6} {'Count':>7} {'FP':>4} {'FP':>4} {'TB':>5} " + f"{'Serial':<16} {'Device':<16} {'Disk':<10}") + print("-" * 95) + + query = """ + SELECT + device_path, + serial_number, + temperature_celsius, + power_on_days, + error_count, + failure_probability_v12_pct, + failure_probability_v13_pct, + size_tb, + disk_role + FROM smart_summary + ORDER BY + CASE disk_role + WHEN 'parity' THEN 1 + WHEN '2-parity' THEN 2 + ELSE 3 + END, + device_path + """ + + with self.pg_conn.cursor() as cur: + cur.execute(query) + for row in cur.fetchall(): + temp = row['temperature_celsius'] or 0 + days = row['power_on_days'] or 0 + errors = row['error_count'] or 0 + fp_v12 = int(row['failure_probability_v12_pct'] or 0) + fp_v13 = int(row['failure_probability_v13_pct'] or 0) + size = row['size_tb'] or 0 + serial = (row['serial_number'] or '')[:16] + device = (row['device_path'] or '')[:16] + role = row['disk_role'] or '-' + + print(f"{temp:5d} {days:6d} {errors:7d} {fp_v12:3d}% {fp_v13:3d}% {size:5.1f} " + f"{serial:<16} {device:<16} {role:<10}") + + print() + + def run(self): + """Main execution flow.""" + try: + self.connect_influxdb() + self.connect_postgres() + + # Load device metadata from PostgreSQL + self.load_device_metadata() + + # Fetch SMART data from InfluxDB + devices_data = self.fetch_smart_data_from_influx() + + # Process and calculate failure probabilities + processed = self.process_device_data(devices_data) + + # Save to PostgreSQL + self.save_to_postgres(processed) + + # Print SnapRAID-style summary + self.print_summary() + + except Exception as e: + print(f"ERROR: {e}") + import traceback + traceback.print_exc() + sys.exit(1) + + finally: + if self.influx_client: + self.influx_client.close() + if self.pg_conn: + self.pg_conn.close() + + +def main(): + # Validate environment variables + required_vars = ['INFLUXDB_URL', 'INFLUXDB_TOKEN', 'POSTGRES_PASSWORD'] + missing = [var for var in required_vars if not os.getenv(var)] + + if missing: + print(f"ERROR: Missing required environment variables: {', '.join(missing)}") + print("Please create a .env file based on .env.example") + sys.exit(1) + + logger = SmartLogger() + logger.run() + + +if __name__ == '__main__': + main() diff --git a/test_attr_193.py b/test_attr_193.py new file mode 100644 index 0000000..c3eef0c --- /dev/null +++ b/test_attr_193.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +"""Test attribute 193 calculation""" + +import math + +SMART_193_R = [ + 0.0000, 0.0016, 0.0032, 0.0036, 0.0039, + 0.0042, 0.0046, 0.0049, 0.0052, 0.0054, + 0.0057, 0.0060, 0.0062, 0.0065, 0.0068, + 0.0071, 0.0074, 0.0077, 0.0080, 0.0083, + 0.0086, 0.0091, 0.0094, 0.0098, 0.0101, + 0.0104, 0.0108, 0.0111, 0.0119, 0.0122, + 0.0127, 0.0130, 0.0134, 0.0137, 0.0141, + 0.0144, 0.0146, 0.0152, 0.0155, 0.0159, + 0.0163, 0.0165, 0.0168, 0.0172, 0.0176, + 0.0179, 0.0184, 0.0188, 0.0190, 0.0194, + 0.0197, 0.0201, 0.0204, 0.0207, 0.0209, + 0.0213, 0.0215, 0.0219, 0.0221, 0.0225, + 0.0229, 0.0234, 0.0241, 0.0246, 0.0253, + 0.0263, 0.0278, 0.0286, 0.0293, 0.0298, + 0.0302, 0.0306, 0.0311, 0.0315, 0.0319, + 0.0322, 0.0329, 0.0334, 0.0338, 0.0343, + 0.0348, 0.0352, 0.0358, 0.0362, 0.0367, + 0.0371, 0.0374, 0.0378, 0.0383, 0.0388, + 0.0393, 0.0397, 0.0401, 0.0404, 0.0410, + 0.0416, 0.0422, 0.0428, 0.0436, 0.0443, + 0.0449, 0.0454, 0.0457, 0.0462, 0.0468, + 0.0473, 0.0479, 0.0483, 0.0488, 0.0491, + 0.0493, 0.0497, 0.0500, 0.0504, 0.0507, + 0.0510, 0.0514, 0.0519, 0.0523, 0.0528, + 0.0533, 0.0538, 0.0542, 0.0547, 0.0551, + 0.0556, 0.0560, 0.0565, 0.0572, 0.0577, + 0.0584, 0.0590, 0.0594, 0.0599, 0.0603, + 0.0607, 0.0611, 0.0616, 0.0621, 0.0626, + 0.0632, 0.0639, 0.0647, 0.0655, 0.0661, + 0.0669, 0.0676, 0.0683, 0.0691, 0.0699, + 0.0708, 0.0713, 0.0719, 0.0724, 0.0730, + 0.0736, 0.0745, 0.0751, 0.0759, 0.0769, + 0.0779, 0.0787, 0.0796, 0.0804, 0.0815, + 0.0825, 0.0833, 0.0840, 0.0847, 0.0854, + 0.0859, 0.0865, 0.0873, 0.0881, 0.0890, + 0.0900, 0.0912, 0.0919, 0.0929, 0.0942, + 0.0956, 0.0965, 0.0976, 0.0986, 0.0995, + 0.1006, 0.1019, 0.1031, 0.1038, 0.1045, + 0.1051, 0.1058, 0.1066, 0.1072, 0.1077, + 0.1084, 0.1091, 0.1099, 0.1104, 0.1111, + 0.1118, 0.1127, 0.1135, 0.1142, 0.1149, + 0.1157, 0.1163, 0.1168, 0.1173, 0.1179, + 0.1184, 0.1189, 0.1195, 0.1203, 0.1208, + 0.1213, 0.1223, 0.1231, 0.1240, 0.1246, + 0.1252, 0.1260, 0.1269, 0.1276, 0.1287, + 0.1303, 0.1311, 0.1319, 0.1328, 0.1335, + 0.1341, 0.1348, 0.1362, 0.1373, 0.1380, + 0.1387, 0.1392, 0.1398, 0.1403, 0.1408, + 0.1412, 0.1418, 0.1422, 0.1428, 0.1434, + 0.1439, 0.1445, 0.1451, 0.1457, 0.1464, + 0.1469, 0.1475, 0.1480, 0.1486, 0.1491, + 0.1498, +] + +SMART_193_STEP = 649 + +# User's drive data +raw_value = 150091 + +# Calculate table index +table_index = raw_value // SMART_193_STEP +print(f"Attribute 193 RAW_VALUE: {raw_value}") +print(f"STEP: {SMART_193_STEP}") +print(f"Table index: {raw_value} // {SMART_193_STEP} = {table_index}") + +# Get monthly rate +if table_index >= len(SMART_193_R): + table_index = len(SMART_193_R) - 1 + +monthly_rate = SMART_193_R[table_index] +print(f"Monthly failure rate (SMART_193_R[{table_index}]): {monthly_rate}") + +# Annualize +afr = (365.0 / 30.0) * monthly_rate +print(f"Annualized: (365/30) * {monthly_rate} = {afr:.4f}") + +# Apply Poisson +prob = (1 - math.exp(-afr)) * 100 +print(f"Poisson P(failure): (1 - e^(-{afr:.4f})) * 100 = {prob:.2f}%") + +print("\n" + "="*60) +print(f"SnapRAID reports: 81%") +print(f"Our calculation: {prob:.2f}%") +print("="*60) diff --git a/test_backblaze_calc.py b/test_backblaze_calc.py new file mode 100644 index 0000000..ec529b0 --- /dev/null +++ b/test_backblaze_calc.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +"""Test BackBlaze calculation with different values""" + +import math +from backblaze_tables import SMART_5_R, SMART_187_R, SMART_188_R, SMART_197_R, SMART_198_R + +def poisson_prob(rate): + """Calculate P(at least 1 failure) = 1 - e^(-rate)""" + return 1 - math.exp(-rate) + +def test_calculation(attr_value): + """Test with a given attribute value""" + # Scale monthly to annual (like snapraid does) + afr_5 = (365.0 / 30.0) * SMART_5_R[min(attr_value, 255)] + afr_187 = (365.0 / 30.0) * SMART_187_R[min(attr_value, 255)] + afr_188 = (365.0 / 30.0) * SMART_188_R[min(attr_value, 255)] + afr_197 = (365.0 / 30.0) * SMART_197_R[min(attr_value, 255)] + afr_198 = (365.0 / 30.0) * SMART_198_R[min(attr_value, 255)] + + # Take maximum (like snapraid) + max_afr = max(afr_5, afr_187, afr_188, afr_197, afr_198) + + # Apply Poisson + prob = poisson_prob(max_afr) * 100 + + return max_afr, prob + +print("Testing different attribute values:") +print("=" * 80) + +for test_val in [0, 50, 100, 150, 200, 255]: + afr, prob = test_calculation(test_val) + print(f"Attribute value {test_val:3d}: AFR={afr:.4f}, Probability={prob:.2f}%") + +print("\n" + "=" * 80) +print("What we're seeing from /dev/sdc:") +print(" All critical attributes: RAW_VALUE=0, VALUE=100") +print(" SnapRAID reports: 81%") +print(" Our calculation with value=0: ", end="") +afr0, prob0 = test_calculation(0) +print(f"{prob0:.2f}%") +print(" Our calculation with value=100:", end="") +afr100, prob100 = test_calculation(100) +print(f"{prob100:.2f}%")