wp931120/tiny_agent/data-analysis

import pandas as pd
# CSV
df = pd.read_csv('data.csv')
# Excel
df = pd.read_excel('data.xlsx', sheet_name='Sheet1')
# JSON
df = pd.read_json('data.json')
# SQL
import sqlite3
conn = sqlite3.connect('database.db')
df = pd.read_sql('SELECT * FROM table', conn)

# Basic info
df.info()
df.describe()
df.head()
# Check missing values
df.isnull().sum()
# Check duplicates
df.duplicated().sum()

# Handle missing values
df.fillna(df.mean(), inplace=True)  # Numeric: mean
df.fillna(df.mode()[0], inplace=True)  # Categorical: mode
df.dropna(subset=['important_col'], inplace=True)  # Drop if critical
# Remove duplicates
df.drop_duplicates(inplace=True)
# Fix data types
df['date'] = pd.to_datetime(df['date'])
df['amount'] = pd.to_numeric(df['amount'], errors='coerce')
# Standardize text
df['category'] = df['category'].str.lower().str.strip()
# Handle outliers
from scipy import stats
df = df[(np.abs(stats.zscore(df.select_dtypes(include=[np.number]))) < 3).all(axis=1)]

import matplotlib.pyplot as plt
import seaborn as sns
# Numeric: histogram
df['column'].hist(bins=30)
plt.show()
# Categorical: bar chart
df['category'].value_counts().plot(kind='bar')
plt.show()

# Numeric vs Numeric: scatter plot
df.plot.scatter(x='var1', y='var2')
plt.show()
# Correlation heatmap
sns.heatmap(df.corr(), annot=True, cmap='coolwarm')
plt.show()
# Categorical vs Numeric: box plot
df.boxplot(column='numeric_col', by='category')
plt.show()

# Pair plot
sns.pairplot(df)
plt.show()
# Grouped analysis
df.groupby('category').agg({
    'numeric1': 'mean',
    'numeric2': 'sum'
})

from scipy import stats
# T-test: compare two groups
group1 = df[df['category'] == 'A']['value']
group2 = df[df['category'] == 'B']['value']
t_stat, p_value = stats.ttest_ind(group1, group2)
# Chi-square test: categorical association
contingency_table = pd.crosstab(df['cat1'], df['cat2'])
chi2, p_value, dof, expected = stats.chi2_contingency(contingency_table)
# ANOVA: compare multiple groups
groups = [df[df['category'] == cat]['value'] for cat in df['category'].unique()]
f_stat, p_value = stats.f_oneway(*groups)

# Pearson correlation (linear)
corr = df['var1'].corr(df['var2'])
# Spearman correlation (non-linear)
corr = df['var1'].corr(df['var2'], method='spearman')

# Set datetime index
df['date'] = pd.to_datetime(df['date'])
df.set_index('date', inplace=True)
# Resample
monthly = df.resample('M').sum()
daily_mean = df.resample('D').mean()
# Rolling statistics
df['rolling_mean'] = df['value'].rolling(window=7).mean()
df['rolling_std'] = df['value'].rolling(window=7).std()
# Decomposition
from statsmodels.tsa.seasonal import seasonal_decompose
decomposition = seasonal_decompose(df['value'], model='additive', period=12)
decomposition.plot()
plt.show()

from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import classification_report, confusion_matrix
# Prepare features
X = df[['feature1', 'feature2', 'feature3']]
y = df['target']
# Split data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
# Train model
model = RandomForestClassifier(random_state=42)
model.fit(X_train, y_train)
# Evaluate
y_pred = model.predict(X_test)
print(classification_report(y_test, y_pred))
# Feature importance
feature_importance = pd.DataFrame({
    'feature': X.columns,
    'importance': model.feature_importances_
}).sort_values('importance', ascending=False)

python scripts/auto_analysis.py data.csv

python scripts/profile_data.py data.csv

# Structure analysis notebooks/scripts as:
# 1. Imports and setup
# 2. Data loading
# 3. Data profiling
# 4. Data cleaning
# 5. EDA
# 6. Statistical analysis
# 7. Advanced analysis (if needed)
# 8. Conclusions and recommendations

# Data Analysis Report
 
## Executive Summary
-Key findings (3-5 bullet points)
 
## Dataset Overview
-Shape, features, data types
-Quality issues identified
 
## Key Insights
### Insight 1: [Title]
-Observation
-Evidence (chart/statistic)
-Business implication
 
### Insight 2: [Title]
...
 
## Recommendations
1.[Actionable recommendation based on data]
2....
 
## Appendix
-Detailed statistics
-All visualizations

# Load
df = pd.read_csv('sales.csv')
df['date'] = pd.to_datetime(df['date'])
# Analyze by time
daily_sales = df.groupby('date')['revenue'].sum()
monthly_sales = df.resample('M', on='date')['revenue'].sum()
# Top products
top_products = df.groupby('product')['revenue'].sum().sort_values(ascending=False).head(10)
# Growth rate
df['growth_rate'] = df.groupby('product')['revenue'].pct_change()

from sklearn.preprocessing import StandardScaler
from sklearn.cluster import KMeans
# Prepare features
features = df[['age', 'income', 'spending_score']]
scaler = StandardScaler()
features_scaled = scaler.fit_transform(features)
# Clustering
kmeans = KMeans(n_clusters=4, random_state=42)
df['segment'] = kmeans.fit_predict(features_scaled)
# Analyze segments
segment_profile = df.groupby('segment').agg({
    'age': 'mean',
    'income': 'mean',
    'spending_score': 'mean',
    'customer_id': 'count'
})

from scipy import stats
# Group data
control = df[df['group'] == 'control']['conversion']
treatment = df[df['group'] == 'treatment']['conversion']
# Statistical test
t_stat, p_value = stats.ttest_ind(control, treatment)
# Effect size
effect_size = (treatment.mean() - control.mean()) / control.mean()
# Confidence interval
from scipy.stats import t
mean_diff = treatment.mean() - control.mean()
se = np.sqrt(treatment.var()/len(treatment) + control.var()/len(control))
ci = t.interval(0.95, len(treatment)+len(control)-2, loc=mean_diff, scale=se)