机器学习基础

import numpy as np

score = np.array(
[[80, 89, 86, 67, 79],
[78, 97, 89, 67, 81],
[90, 94, 78, 67, 74],
[91, 91, 90, 67, 69],
[76, 87, 75, 67, 86],
[70, 79, 84, 67, 84],
[94, 92, 93, 67, 64],
[86, 85, 83, 67, 80]])

print(score)

# 接着前面的代码运行
print(f"score的形状为{score.shape}")
print(f"score的维度为{score.ndim}")
print(f"score的元素个数为{score.size}")
print(f"score的元素大小为{score.itemsize}")
print(f"score的内存大小为{score.nbytes}")
print(f"score的元素类型为{score.dtype}")

ones = np.ones([4,8])
ones

import numpy as np

a = np.array([[1, 2, 3], [4, 5, 6]])
a1 = np.array(a)
a2 = np.asarray(a)

print(a1)
print(a2)

print("-" * 20)
a[0][0] = 0

print(a1)
print(a2)

import numpy as np

a1 = np.linspace(1, 10, 2)
print(a1)

a2 = np.logspace(1, 4, 4, base=2)
print(a2)

a3 = np.arange(1, 10, 2)
print(a3)

import numpy as np

# 生成一个 [0.0, 1.0) 之间的均匀分布的随机浮点数
rand_num = np.random.rand()
print("均匀分布的随机浮点数:", rand_num)

# 生成一个形状为(3, 2)的均匀分布的随机浮点数组
rand_array = np.random.rand(3, 2)
print("均匀分布的随机数组:\n", rand_array)

# 生成一个 [0, 100) 之间的均匀分布的随机整数数组
rand_num = np.random.randint(0, 100, size=(3, 2))
print("均匀分布的随机整数数组:\n", rand_num)

import numpy as np

# 创建一个 3x4 的数组
arr = np.array([[1, 2, 3, 4],
                [5, 6, 7, 8],
                [9, 10, 11, 12]])

# 提取第 1 行和第 2 行的所有列
sub_array = arr[0:2, :]
print("第1行和第2行的所有列:\n", sub_array)

# 提取第 2 列和第 3 列的所有行
sub_array = arr[:, 1:3]
print("第2列和第3列的所有行:\n", sub_array)

# 提取第 1 行第 2 列到第 3 列的元素
sub_array = arr[0, 1:3]
print("第1行第2列到第3列的元素:", sub_array)

import numpy as np

array = np.arange(10)
array = array.reshape(2, 5)
print(array)

array.resize(5, 2)
print(array)

print(array.T)

import numpy as np

array = np.arange(10)
print(f"修改前的dtype: {array.dtype}")

array = array.astype(np.float32)
print(f"修改后的dtype: {array.dtype}")

print("-" * 20)

arr = np.array([[[1, 2, 3], [4, 5, 6]], [[12, 3, 34], [5, 6, 7]]])
print(f"数组: {arr}, 形状: {arr.shape}, 类型: {arr.dtype}")
bytes = arr.tobytes()
print(f"转成字节后的大小: {len(bytes)}")

arr = np.frombuffer(bytes, dtype=np.int64).reshape(2, 2, 3)
print(f"从字节转成数组: {arr}")

import numpy as np

array = np.array([[1, 1, 2, 3], [4, 5, 5, 6]])
array = np.unique(array)
print(f"去重后的数组: {array}")

import numpy as np

array1 = np.array([1, 2, 3, 4, 5, 6])
array2 = np.array([2, 3, 4, 5, 6, 7])

print(f"array1 + array2 = {array1 + array2}")
print(f"array1 * array2 = {array1 * array2}")
print(f"array1 ** 2 = {array1 ** 2}")
print(f"array1和array2的矩阵相乘 = {array1 @ array2}")

import numpy as np

score = np.random.randint(40, 100, (3, 5))
print(f"随机生成的成绩: {score}")

print(f"大于等于60的成绩: {score[score >= 60]}")
print(f"每个成绩和60比较: {score >= 60}")
score[score >= 60] = 1
print(f"将大于等于60的成绩替换为1: {score}")

import numpy as np

array = np.array([1, 2, 3, 4, 5, 6])

print(f"所有元素都大于5: {np.all(array >= 5)}")
print(f"所有元素都小于6: {np.all(array <= 6)}")
print(f"至少有一个元素大于5: {np.any(array > 5)}")

# 判断前四名学生, 前四门课程中, 成绩中大于 60 的置为 1, 否则为 0
temp = score[:4, :4]
np.where(temp > 60, 1, 0)

# 判断前四名学生, 前四门课程中, 成绩中大于 60 且小于 90 的换为 1, 否则为 0
np.where(np.logical_and(temp > 60, temp < 90), 1, 0)

# 判断前四名学生, 前四门课程中, 成绩中大于 90 或小于 60 的换为 1, 否则为 0
np.where(np.logical_or(temp > 90, temp < 60), 1, 0)

arr1 = np.array([[0],[1],[2],[3]])  # 4 x 1
arr1.shape
# (4, 1)

arr2 = np.array([1,2,3])  # 1 x 3
arr2.shape
# (3,)

arr1+arr2
# 结果是:
array([[1, 2, 3],
       [2, 3, 4],
       [3, 4, 5],
       [4, 5, 6]])

import numpy as np

array = np.array([[1, 2, 3], [4, 5, 6]])
print(f"数组的最大值: {np.max(array)}")
print(f"数组的最小值: {np.min(array)}")
print(f"数组的中位数: {np.median(array)}")
print(f"数组的平均值: {np.mean(array)}")
print(f"数组的标准差: {np.std(array)}")
print(f"数组的方差: {np.var(array)}")
print(f"最小元素的下标: {np.argmin(array)}")
print(f"最大元素的下标: {np.argmax(array)}")

# 导入
import pandas as pd

# 从 CSV 文件读取数据集
df = pd.read_csv("data.csv")
print("查看数据集:", df, sep="\n", end="\n\n")
"""
      year   country           GDP
0     1960        美国  543300000000
1     1960        英国   73233967692
2     1960        法国   62225478000
3     1960        中国   59716467625
4     1960        日本   44307342950
...    ...       ...           ...
9925  2019  圣多美和普林西比     418637388
9926  2019        帕劳     268354919
9927  2019      基里巴斯     194647201
9928  2019        瑙鲁     118223430
9929  2019       图瓦卢      47271463

[9930 rows x 3 columns]
"""

# 查看前五行
print("查看前五行:", df.head(), sep="\n", end="\n\n")
"""
   year country           GDP
0  1960      美国  543300000000
1  1960      英国   73233967692
2  1960      法国   62225478000
3  1960      中国   59716467625
4  1960      日本   44307342950
"""

# 查询中国 GDP
print("查询中国GDP:", df[df["country"] == "中国"].head(10), sep="\n", end="\n\n")
"""
     year country          GDP
3    1960      中国  59716467625
107  1961      中国  50056868957
211  1962      中国  47209359005
316  1963      中国  50706799902
421  1964      中国  59708343488
525  1965      中国  70436266146
639  1966      中国  76720285969
755  1967      中国  72881631326
874  1968      中国  70846535055
993  1969      中国  79705906247
"""

# 将 year 设为索引
new_df = df.set_index("year")
print("将year设为索引:", new_df, sep="\n", end="\n\n")
"""
       country           GDP
year
1960        美国  543300000000
1960        英国   73233967692
1960        法国   62225478000
1960        中国   59716467625
1960        日本   44307342950
...        ...           ...
2019  圣多美和普林西比     418637388
2019        帕劳     268354919
2019      基里巴斯     194647201
2019        瑙鲁     118223430
2019       图瓦卢      47271463

[9930 rows x 2 columns]
"""

# 画出中国和日本的 GDP 折线图
jp_gdp = df[df.country=='日本'].set_index('year') # 按条件选取数据后, 重设索引
jp_gdp.GDP.plot()
china_gdp = df[df.country=='中国'].set_index('year') # 按条件选取数据后, 重设索引
china_gdp.GDP.plot()

import pandas as pd

# 使用默认自增索引
series1 = pd.Series([1, 2, 3, 4, 5])
print(series1)

# 自定义索引
series2 = pd.Series([1, 2, 3, 4, 5], index=['a', 'b', 'c', 'd', 'e'])
print(series2)

import pandas as pd

# 使用元组创建
series1 = pd.Series((1, 2, 3, 4, 5))
print(series1)

# 使用字典创建
series2 = pd.Series({
    'a': 1,
    'b': 2,
    'c': 3,
    'd': 4,
    'e': 5
})
print(series2)

import pandas as pd
import numpy as np

ndarray = np.array([1, 2, 3])
print(ndarray)
print(type(ndarray))

series = pd.Series(ndarray)
print(series)
print(type(series))

import pandas as pd
import numpy as np

ndarray = np.array([1, 2, 3, 4])
series = pd.Series(ndarray)

print(f"索引: {series.index}")
print(f"元素: {series.values}")
print(f"数据类型: {series.dtypes}")
print(f"形状: {series.shape}")
print(f"元素数量: {series.size}")
print(f"维度: {series.ndim}")
print(f"名称: {series.name}")
print(f"轴信息: {series.axes}")
print(f"数据占用字节数: {series.nbytes}")
print(f"内部标志: {series.flags}")

import pandas as pd
import numpy as np

ndarray = np.array(["Apple", "Banana", "Cherry", "Date"])
series = pd.Series(ndarray, index=["a", "b", "c", "d"])

print(series.a) # Apple
print(series["b"]) # Banana

import pandas as pd

df = pd.read_csv("data.csv")
print(df)
print(type(df))

import pandas as pd

df1_data = {
    '日期': ['2021-08-21', '2021-08-22', '2021-08-23'],
    '温度': [25, 26, 50],
    '湿度': [81, 50, 56]
}
df1 = pd.DataFrame(data=df1_data)
print(df1)

import pandas as pd

df2_data = [
    ('2021-08-21', 25, 81),
    ('2021-08-22', 26, 50),
    ('2021-08-23', 27, 56)
]

df2 = pd.DataFrame(
    data=df2_data,
    columns=['日期', '温度', '湿度'],
    index = ['row_1','row_2','row_3'] # 手动指定索引
)
print(df2)

import pandas as pd
import numpy as np

score = np.random.randint(40, 100, (3, 2))
score_df = pd.DataFrame(score, index=["Row1", "Row2", "Row3"], columns=["Column1", "Column2"])
print(score_df)

import pandas as pd
import numpy as np

score = np.random.randint(40, 100, (5, 3))
score_df = pd.DataFrame(score, index=["Row1", "Row2", "Row3", "Row4", "Row5"], columns=["Column1", "Column2", "Column3"])
print(f"行列数: {score_df.shape}")
print(f"行索引对象: {score_df.index}")
print(f"列索引对象: {score_df.columns}")
print(f"值: {score_df.values}")
print(f"转置: {score_df.T}")
print(f"前三行: {score_df.head(3)}")
print(f"后两行: {score_df.tail(2)}")

import pandas as pd
import numpy as np

score = np.random.randint(40, 100, (5, 3))
score_df = pd.DataFrame(score, index=["Row1", "Row2", "Row3", "Row4", "Row5"], columns=["Column1", "Column2", "Column3"])
print(score_df["Column3"])
print(score_df["Column1"]["Row1"])

# 错误的操作: score_df["Row1"]["Column1"]
# 错误的操作: score_df[0][0]

import pandas as pd
import numpy as np

score = np.random.randint(40, 100, (5, 3))
score_df = pd.DataFrame(score, index=["Row1", "Row2", "Row3", "Row4", "Row5"], columns=["Column1", "Column2", "Column3"])

print(score_df.loc["Row1"])
print(score_df.loc["Row1", "Column2"])
print(score_df.loc["Row1":"Row3", "Column2"])

print(score_df.iloc[0])
print(score_df.iloc[0, 1])
print(score_df.iloc[0:3, 1])

import pandas as pd
import numpy as np

score = np.random.randint(40, 100, (5, 3))
score_df = pd.DataFrame(score, index=["Row1", "Row2", "Row3", "Row4", "Row5"], columns=["Column1", "Column2", "Column3"])

score_df["Column1"] = 1
print(score_df)

score_df["Column4"] = score_df["Column1"] + score_df["Column2"] + score_df["Column3"]
print(score_df)

# score_df["Column1", "Row1"] = -100  # 错误
# score_df["Column1"]["Row1"] = -100  # 错误
score_df.loc["Row1", "Column1"] = -100
print(score_df)

import pandas as pd
import numpy as np

score = np.random.randint(40, 100, (5, 3))
score_df = pd.DataFrame(score, index=["Row1", "Row2", "Row3", "Row4", "Row5"], columns=["Column1", "Column2", "Column3"])

print(score_df.index)
print(score_df.columns)

score_df.index = ["行1", "行2", "行3", "行4", "行5"]
score_df.columns = ["列A", "列B", "列C"]

print(score_df.index)
print(score_df.columns)

import pandas as pd
import numpy as np

score = np.random.randint(40, 100, (5, 3))
score_df = pd.DataFrame(score, index=["Row1", "Row2", "Row3", "Row4", "Row5"], columns=["Column1", "Column2", "Column3"])

print(score_df)

score_df.set_index("Column1", inplace=True)

print(score_df)

import pandas as pd

# 构造一个索引和数据都"故意打乱"的DataFrame
df = pd.DataFrame({
    "姓名": ["张三", "李四", "王五", "赵六"],
    "分数": [88, 92, 75, 92],
    "年龄": [20, 19, 21, 18]
}, index=[3, 1, 4, 2])

print("原始DataFrame: ")
print(df)

df['等级'] = [10, 9, 10, 9]
print("添加新列后的DataFrame: ")
print(df)

import pandas as pd

# 构造一个索引和数据都"故意打乱"的DataFrame
df = pd.DataFrame({
    "姓名": ["张三", "李四", "王五", "赵六"],
    "分数": [88, 92, 75, 92],
    "年龄": [20, 19, 21, 18]
}, index=[3, 1, 4, 2])

print("原始DataFrame: ")
print(df)

df['等级'] = [10, 9, 10, 9]
print("添加新列后的DataFrame: ")
print(df)

df.drop(columns=['等级'], inplace=True)
print("删除等级列后的DataFrame: ")
print(df)

import pandas as pd

data = {
    "姓名": ["张三", "李四", "王五"],
    "分数": [88, 92, 75]
}

df = pd.DataFrame(data, index=[3, 1, 2])

print("原始DataFrame: ")
print(df)

df_sorted = df.sort_index()

print("\n按索引排序后: ")
print(df_sorted)

import pandas as pd

df = pd.DataFrame({
    "姓名": ["张三", "李四", "王五", "赵六"],
    "分数": [88, 92, 75, 92],
    "年龄": [20, 19, 21, 18]
}, index=[3, 1, 4, 2])

print("原始DataFrame: ")
print(df)

# ======================
# 按索引排序
# ======================
df_by_index = df.sort_index()

print("\n按索引排序 sort_index(): ")
print(df_by_index)

# ======================
# 按单列值排序
# ======================
df_by_score = df.sort_values(by="分数")

print("\n按分数排序 sort_values(by='分数'): ")
print(df_by_score)

# ======================
# 按多列值排序
# ======================
df_multi = df.sort_values(
    by=["分数", "年龄"],
    ascending=[False, True]
)

print("\n先按分数降序, 再按年龄升序: ")
print(df_multi)

import pandas as pd

# 构造一个索引和数据都"故意打乱"的DataFrame
df = pd.DataFrame({
    "姓名": ["张三", "李四", "王五", "赵六"],
    "分数": [88, 92, 75, 92],
    "年龄": [20, 19, 21, 18]
}, index=[3, 1, 4, 2])

print(df.describe())
df.info()

import pandas as pd

df1 = pd.DataFrame({
    "Col1": [1, 2, 3, 4, 5],
    "Col2": [6, 7, 8, 9, 10],
    "Col3": [11, 12, 13, 14, 15]
})
df2 = pd.DataFrame({
    "Col1": [16, 17, 18, 19, 20],
    "Col2": [21, 22, 23, 24, 25],
    "Col3": [26, 27, 28, 29, 30]
})
print(f"df1.add(df2) = \n{df1.add(df2)}")
print(f"df1 + df2 = \n{df1 + df2}")

print(f"df1.sub(df2) = \n{df1.sub(df2)}")
print(f"df1 - df2 = \n{df1 - df2}")

print(f"df1.mul(df2) = \n{df1.mul(df2)}")
print(f"df1 * df2 = \n{df1 * df2}")

print(f"df1.div(df2) = \n{df1.div(df2)}")
print(f"df1 / df2 = \n{df1 / df2}")

import pandas as pd

df1 = pd.DataFrame({
    "Col1": [1, 2, 3, 4, 5],
    "Col2": [6, 7, 8, 9, 10],
    "Col3": [11, 12, 13, 14, 15]
})
print(f"大于等于5的元素: \n{df1 >= 5}")
print(f"筛选出Col1中值大于等于3的行: \n{df1[df1['Col1'] >= 3]}")

print(f"应用大于等于返回的元素类型: {type(df1 >= 5)}")
print(f"应用大于等于返回的元素类型: {type(df1[df1['Col1'] >= 3])}")

print(f"筛选出Col2中值大于等于8小于等于10的行: \n{df1[(df1['Col2'] >= 8) & (df1['Col2'] <= 10)]}")

import pandas as pd

df1 = pd.DataFrame({
    "Col1": [1, 2, 3, 4, 5],
    "Col2": [6, 7, 8, 9, 10],
    "Col3": [11, 12, 13, 14, 15]
})
print(f"筛选出Col1中值大于等于3的行: \n{df1.query('Col1 >= 3')}")
print(f"筛选出Col1中值大于等于3的行, 并只保留Col1和Col3列: \n{df1.query('Col1 >= 3')[['Col1', 'Col3']]}")
print(f"筛选出Col2中值大于等于8且Col3中值小于等于13的行: \n{df1.query('Col2 >= 8 and Col3 <= 13')}")

print(f"筛选出Col1具有1和3的行: \n{df1[df1.Col1.isin([1, 3])]}")

import pandas as pd

df1 = pd.DataFrame({
    "Col1": [1, 2, 3, 4, 5],
    "Col2": [6, 7, 8, 9, 10],
    "Col3": [11, 12, 13, 14, 15]
})

print(f"df1的描述统计信息: \n{df1.describe()}\n")
print("df1的信息摘要:")
df1.info()
print(f"\ndf1.Col1的最大值: {df1.Col1.max()}")

import pandas as pd

df1 = pd.DataFrame({
    "Col1": [1, 2, 3, 4, 5],
    "Col2": [6, 7, 8, 9, 10],
    "Col3": [11, 12, 13, 14, 15]
}, index=["索引1", "索引2", "索引3", "索引4", "索引5"])

print(f"最大值的索引: {df1.Col1.idxmax()}")
print(f"最小值的索引: {df1.Col1.idxmin()}")

import pandas as pd

df = pd.DataFrame({
    "Col1": [1, 2, 3, 4, 5],
    "Col2": [6, 7, 8, 9, 10],
    "Col3": [11, 12, 13, 14, 15]
}, index=["索引1", "索引2", "索引3", "索引4", "索引5"])

print(f"筛选出每列的最大值: \n{df.apply(lambda x: x.max(), axis=0)}")

import pandas as pd
import os

df = pd.read_csv("data.csv")

print(df)

df.head(10).to_csv("./data_head.csv", index=False)

import pandas as pd
import sqlalchemy

import pymysql

df = pd.DataFrame({
    "name": ["张三", "李四", "王五", "赵六"],
    "age": [20, 21, 22, 23],
    "gender": ["男", "女", "男", "女"]
})

# 创建连接对象
conn = sqlalchemy.create_engine("mysql+pymysql://root:imqi1%40db@localhost/test")

df.to_sql("test_df", conn, if_exists="replace", index=False)

import pandas as pd
import sqlalchemy

conn = sqlalchemy.create_engine("mysql+pymysql://root:imqi1%40db@localhost/test")

df = pd.read_sql("select * from test_df", conn)

print(df)

import pandas as pd

old_df = pd.DataFrame(
    {
        "Col1": [1, 2, 3, 3, 4],
        "Col2": [1, 3, 3, 3, 5],
        "Col3": [2, 2, 3, 3, 5],
        "Col4": [0, 1, 4, 4, 6],
    },
    index=["Row1", "Row2", "Row3", "Row4", "Row5"],
)

new_df = old_df.copy()

new_df["Col1"] = "NewValue"
new_df["Col3"] = new_df["Col2"]
new_df["Col4"] = new_df["Col4"] + 1
new_df["Col5"] = [3, 4, 5, 5, 7]

print(f"old_df: \n{old_df}")
print(f"new_df: \n{new_df}")

import pandas as pd

old_df = pd.DataFrame({
    "Col1": [1, 2, 3],
    "Col2": [4, 5, 6],
    "Col3": [7, 8, 9],
}, index=["Row1", "Row2", "Row3"])

new_df = old_df.copy()

new_df = new_df.assign(Col4="new_val", Col5=[10, 11, 12], Col6=lambda x: x["Col1"] + x["Col2"])

print(f"old_df:\n{old_df}")
print(f"new_df:\n{new_df}")

import pandas as pd

old_df = pd.DataFrame(
    {
        "Col1": [1, 2, 3],
        "Col2": [4, 5, 6],
        "Col3": [7, 8, 9],
    },
    index=["Row1", "Row2", "Row3"],
)

new_df = old_df.copy()

new_df.replace(1, "a", inplace=True)
new_df.replace([2, 3], ["b", "c"], inplace=True)

print(f"old_df: \n{old_df}")
print(f"new_df: \n{new_df}")

import pandas as pd

old_df = pd.DataFrame({
    "Col1": [1, 2, 3, 3, 4],
    "Col2": [1, 3, 3, 4, 5],
    "Col3": [2, 2, 3, 4, 5],
    "Col4": [0, 1, 4, 5, 6],
    "Col5": [3, 4, 5, 6, 7],
}, index=["Row1", "Row2", "Row3", "Row4", "Row5"])

new_df = old_df.copy()

print(f"删除一行: \n{new_df.drop(["Row1"])}")
print(f"删除多行: \n{new_df.drop(["Row1", "Row2"])}")
print(f"Series 删除 Row1 行: \n{new_df.Col1.drop(["Row1"])}")

import pandas as pd

old_df = pd.DataFrame(
    {
        "Col1": [1, 2, 3, 3, 4],
        "Col2": [1, 3, 3, 4, 5],
        "Col3": [2, 2, 3, 4, 5],
        "Col4": [0, 1, 4, 5, 6],
        "Col5": [3, 4, 5, 6, 7],
    },
    index=["Row1", "Row2", "Row3", "Row4", "Row5"],
)

new_df = old_df.copy()

del new_df["Col4"]

print(f"old_df: \n{old_df}")
print(f"new_df: \n{new_df}")

import pandas as pd

old_df = pd.DataFrame(
    {
        "Col1": [1, 2, 3, 3, 4],
        "Col2": [1, 3, 3, 3, 5],
        "Col3": [2, 2, 3, 3, 5],
        "Col4": [0, 1, 4, 4, 6],
        "Col5": [3, 4, 5, 5, 7],
    },
    index=["Row1", "Row2", "Row3", "Row4", "Row5"],
)

new_df = old_df.copy()

new_df.drop_duplicates(inplace=True)

print(f"old_df: \n{old_df}")
print(f"new_df: \n{new_df}")

import pandas as pd

old_series = pd.Series(["1", "2", "2", "3", "4", "5", "5"])

new_series1 = old_series.copy()
new_series2 = old_series.copy()

new_series1.drop_duplicates(inplace=True)
new_series2 = new_series2.unique()

print(f"old_series: \n{old_series}")
print(f"new_series1: \n{new_series1}")
print(f"new_series2: \n{new_series2}")
print(f"type(new_series2): {type(new_series2)}")