NewLife
/
XCode
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

大数据表，自动生成分表策略，支持年月日小时分表

This commit is contained in:
大石头 2024-07-07 19:03:58 +08:00
parent 38d230ee98
commit 12bd6c7f17
10 changed files with 75 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Doc/BigData.md
View File

@ -36,8 +36,8 @@
时间字段几种场景：
1. 时间字段。创建更新时间或者业务时间DataTime，支持timeshard:yyyy:yyMMdd，后面两段是分库分表表达式，第二段为空时只分表不分库，即timeshard::yyMMdd。
2. 雪花Id。支持timeshard:yyyy:yyMMdd，后面两段是分库分表表达式，第二段为空时只分表不分库，即timeshard::yyMMdd。
1. 时间字段。创建更新时间或者业务时间DataTime，支持timeshard:yyMMdd:yyyy，后面两段是分库分表表达式，第三段为空时只分表不分库，即timeshard::yyMMdd。
2. 雪花Id。支持timeshard:yyMMdd:yyyy，后面两段是分库分表表达式，第三段为空时只分表不分库，即timeshard::yyMMdd。

59
XCode/Code/EntityBuilder.cs
View File

@ -1076,16 +1076,9 @@ public class EntityBuilder : ClassBuilder
}
// 自动分表
dc = Table.Columns.FirstOrDefault(e => !e.Identity && e.PrimaryKey && e.DataType == typeof(Int64));
if (dc != null)
{
WriteLine("// 按天分表");
WriteLine("//Meta.ShardPolicy = new TimeShardPolicy(nameof({0}), Meta.Factory)", dc.Name);
WriteLine("//{");
WriteLine("// TablePolicy = \"{0}_{1:yyyyMMdd}\",");
WriteLine("// Step = TimeSpan.FromDays(1),");
WriteLine("//};");
}
dc = ScaleColumn;
if (dc != null && !dc.DataScale.IsNullOrEmpty() && dc.DataScale.StartsWithIgnoreCase("timeShard:"))
BuildShardPolicy(dc);
var ns = new HashSet<String>(Table.Columns.Select(e => e.Name), StringComparer.OrdinalIgnoreCase);
WriteLine();
@ -1131,6 +1124,52 @@ public class EntityBuilder : ClassBuilder
WriteLine("}");
}
/// <summary>生成分表策略</summary>
/// <param name="dc"></param>
protected virtual void BuildShardPolicy(IDataColumn dc)
{
if (dc.DataScale.IsNullOrEmpty()) return;
var ss = dc.DataScale.Split(":");
var tablePolicy = ss.Length >= 2 ? ss[1] : "";
var connPolicy = ss.Length >= 3 ? ss[2] : "";
if (!connPolicy.IsNullOrEmpty() || !tablePolicy.IsNullOrEmpty())
{
WriteLine();
if (tablePolicy.Contains("HH") || connPolicy.Contains("HH"))
WriteLine("// 按小时分表");
else if (tablePolicy.Contains("dd") || connPolicy.Contains("dd"))
WriteLine("// 按天分表");
else if (tablePolicy.Contains("MM") || connPolicy.Contains("MM"))
WriteLine("// 按月分表");
else if (tablePolicy.Contains("yy") || connPolicy.Contains("yy"))
WriteLine("// 按年分表");
else
WriteLine("// 按[{0}/{1}]分表", connPolicy, tablePolicy);
WriteLine("Meta.ShardPolicy = new TimeShardPolicy(nameof({0}), Meta.Factory)", dc.Name);
WriteLine("{");
if (!connPolicy.IsNullOrEmpty())
WriteLine($" ConnPolicy = \"{{0}}_{{1:{connPolicy}}}\",");
if (!tablePolicy.IsNullOrEmpty())
WriteLine($" TablePolicy = \"{{0}}_{{1:{tablePolicy}}}\",");
if (tablePolicy.Contains("HH") || connPolicy.Contains("HH"))
WriteLine(" Step = TimeSpan.FromHours(1),");
else if (tablePolicy.Contains("dd") || connPolicy.Contains("dd"))
WriteLine(" Step = TimeSpan.FromDays(1),");
else if (tablePolicy.Contains("MM") || connPolicy.Contains("MM"))
WriteLine(" Step = TimeSpan.FromDays(30),");
else if (tablePolicy.Contains("yy") || connPolicy.Contains("yy"))
WriteLine(" Step = TimeSpan.FromDays(365),");
else
WriteLine(" Step = TimeSpan.FromDays(1),");
WriteLine("};");
}
}
static String[] _validExcludes = ["CreateUser", "CreateUserID", "CreateTime", "CreateIP", "UpdateUser", "UpdateUserID", "UpdateTime", "UpdateIP", "Remark", "TraceId"];
/// <summary>数据验证</summary>
protected virtual void BuildValid()

14
XUnitTest.XCode/Code/Entity/日志.Biz.cs
View File

@ -36,12 +36,14 @@ public partial class Log : Entity<Log>
// 累加字段，生成 Update xx Set Count=Count+1234 Where xxx
//var df = Meta.Factory.AdditionalFields;
//df.Add(nameof(LinkID));
// 按天分表
//Meta.ShardPolicy = new TimeShardPolicy(nameof(ID), Meta.Factory)
//{
// TablePolicy = "{0}_{1:yyyyMMdd}",
// Step = TimeSpan.FromDays(1),
//};
// 按小时分表
Meta.ShardPolicy = new TimeShardPolicy(nameof(ID), Meta.Factory)
{
ConnPolicy = "{0}_{1:yyyyMM}",
TablePolicy = "{0}_{1:yyMMddHH}",
Step = TimeSpan.FromHours(1),
};
// 过滤器 UserModule、TimeModule、IPModule
Meta.Modules.Add(new UserModule { AllowEmpty = false });

10
XUnitTest.XCode/Code/Entity/日志.cs
View File

@ -27,11 +27,11 @@ public partial class Log : ILog, IEntity<ILog>
{
#region 属性
private Int64 _ID;
/// <summary>编号。按天分表</summary>
/// <summary>编号。按小时分表，按月分库</summary>
[DisplayName("编号")]
[Description("编号。按天分表")]
[Description("编号。按小时分表，按月分库")]
[DataObjectField(true, false, false, 0)]
[BindColumn("ID", "编号。按天分表", "", DataScale = "timeShard:yyMMdd")]
[BindColumn("ID", "编号。按小时分表，按月分库", "", DataScale = "timeShard:yyMMddHH:yyyyMM")]
public Int64 ID { get => _ID; set { if (OnPropertyChanging("ID", value)) { _ID = value; OnPropertyChanged("ID"); } } }
private String? _Category;
@ -356,7 +356,7 @@ public partial class Log : ILog, IEntity<ILog>
/// <summary>取得日志字段信息的快捷方式</summary>
public partial class _
{
/// <summary>编号。按天分表</summary>
/// <summary>编号。按小时分表，按月分库</summary>
public static readonly Field ID = FindByName("ID");
/// <summary>类别</summary>
@ -416,7 +416,7 @@ public partial class Log : ILog, IEntity<ILog>
/// <summary>取得日志字段名称的快捷方式</summary>
public partial class __
{
/// <summary>编号。按天分表</summary>
/// <summary>编号。按小时分表，按月分库</summary>
public const String ID = "ID";
/// <summary>类别</summary>

7
XUnitTest.XCode/Code/Entity/用户日志.Biz.cs
View File

@ -35,6 +35,13 @@ public partial class UserLog : Entity<UserLog>
//var df = Meta.Factory.AdditionalFields;
//df.Add(nameof(LinkID));
// 按月分表
Meta.ShardPolicy = new TimeShardPolicy(nameof(DataTime), Meta.Factory)
{
TablePolicy = "{0}_{1:yyMM}",
Step = TimeSpan.FromDays(30),
};
// 过滤器 UserModule、TimeModule、IPModule
Meta.Modules.Add(new UserModule { AllowEmpty = false });
Meta.Modules.Add<TimeModule>();

2
XUnitTest.XCode/Code/EntityInterfaces/ILog.cs
View File

@ -11,7 +11,7 @@ namespace XCode.Membership666;
public partial interface ILog
{
#region 属性
/// <summary>编号。按天分表</summary>
/// <summary>编号。按小时分表，按月分库</summary>
Int64 ID { get; set; }
/// <summary>类别</summary>

2
XUnitTest.XCode/Code/EntityModels/LogModel.cs
View File

@ -11,7 +11,7 @@ namespace XCode.Membership666;
public partial class LogModel : ILog
{
#region 属性
/// <summary>编号。按天分表</summary>
/// <summary>编号。按小时分表，按月分库</summary>
public Int64 ID { get; set; }
/// <summary>类别</summary>

2
XUnitTest.XCode/Code/Member.xml
View File

@ -245,7 +245,7 @@
</Table>
<Table Name="Log" Description="日志" ConnName="Log" InsertOnly="True">
<Columns>
<Column Name="ID" DataType="Int64" PrimaryKey="True" DataScale="timeShard:yyMMdd" Description="编号。按天分表" />
<Column Name="ID" DataType="Int64" PrimaryKey="True" DataScale="timeShard:yyMMddHH:yyyyMM" Description="编号。按小时分表，按月分库" />
<Column Name="Category" DataType="String" Description="类别" />
<Column Name="Action" DataType="String" Description="操作" />
<Column Name="LinkID" DataType="Int32" Description="链接" />

2
XUnitTest.XCode/Code/Member2.xml
View File

@ -245,7 +245,7 @@
</Table>
<Table Name="Log" Description="日志" ConnName="Log" InsertOnly="True">
<Columns>
<Column Name="ID" DataType="Int64" PrimaryKey="True" DataScale="timeShard:yyMMdd" Description="编号。按天分表" />
<Column Name="ID" DataType="Int64" PrimaryKey="True" DataScale="timeShard:yyMMddHH:yyyyMM" Description="编号。按小时分表，按月分库" />
<Column Name="Category" DataType="String" Description="类别" />
<Column Name="Action" DataType="String" Description="操作" />
<Column Name="LinkID" ColumnName="link_id" DataType="Int32" Description="链接" />

6
XUnitTest.XCode/Code/entity_log_normal_biz.cs
View File

@ -37,12 +37,6 @@ public partial class Log : Entity<Log>
// 累加字段，生成 Update xx Set Count=Count+1234 Where xxx
//var df = Meta.Factory.AdditionalFields;
//df.Add(nameof(LinkID));
// 按天分表
//Meta.ShardPolicy = new TimeShardPolicy(nameof(ID), Meta.Factory)
//{
// TablePolicy = "{0}_{1:yyyyMMdd}",
// Step = TimeSpan.FromDays(1),
//};
// 过滤器 UserModule、TimeModule、IPModule
Meta.Modules.Add(new UserModule { AllowEmpty = false });