既然是反序列得到的值，直接赋值就行了。这里不需要SetItem，因为SetItem会设置脏数据，在实体对象很多时（>100w）造成巨大的内存开销。这个修改，导致Area加载csv数据再批量导入数据库时，没有脏数据造成字段数据丢失，因此需要显式指定FullInsert
This commit is contained in:
parent
85284874cb
commit
70df4522c4
|
|
@ -1951,7 +1951,9 @@ public partial class Entity<TEntity> : EntityBase, IAccessor where TEntity : Ent
|
|||
if (!binary.TryRead(fi.Type, ref value)) return false;
|
||||
|
||||
// 顺序要求很高
|
||||
SetItem(fi.Name, value);
|
||||
//SetItem(fi.Name, value);
|
||||
// 既然是反序列得到的值，直接赋值就行了。这里不需要SetItem，因为SetItem会设置脏数据，在实体对象很多时（>100w）造成巨大的内存开销
|
||||
this[fi.Name] = value;
|
||||
}
|
||||
|
||||
return true;
|
||||
|
|
|
|||
|
|
@ -215,9 +215,9 @@ public static class EntityExtension
|
|||
if (session.Dal.SupportBatch && list2.Count() > 1)
|
||||
{
|
||||
// 根据是否来自数据库，拆分为两组
|
||||
var ts = Split(list2);
|
||||
list2 = ts.Item1;
|
||||
rs += BatchSave(session, ts.Item2.Valid(true));
|
||||
var (updates, others) = Split(list2);
|
||||
list2 = updates;
|
||||
rs += BatchSave(session, others.Valid(true), null);
|
||||
}
|
||||
|
||||
return rs + DoAction(list2, useTransition, e => e.Save(), session);
|
||||
|
|
@ -243,15 +243,19 @@ public static class EntityExtension
|
|||
if (session.Dal.SupportBatch && list2.Count() > 1)
|
||||
{
|
||||
// 根据是否来自数据库，拆分为两组
|
||||
var ts = Split(list2);
|
||||
list2 = ts.Item1;
|
||||
rs += BatchSave(session, ts.Item2);
|
||||
var (updates, others) = Split(list2);
|
||||
list2 = updates;
|
||||
rs += BatchSave(session, others, null);
|
||||
}
|
||||
|
||||
return rs + DoAction(list2, useTransition, e => e.SaveWithoutValid(), session);
|
||||
}
|
||||
|
||||
private static Tuple<IList<T>, IList<T>> Split<T>(IEnumerable<T> list) where T : IEntity
|
||||
/// <summary>拆分为来自数据库的更新和其它的Upsert</summary>
|
||||
/// <typeparam name="T"></typeparam>
|
||||
/// <param name="list"></param>
|
||||
/// <returns></returns>
|
||||
private static (IList<T> updates, IList<T> others) Split<T>(IEnumerable<T> list) where T : IEntity
|
||||
{
|
||||
var updates = new List<T>();
|
||||
var others = new List<T>();
|
||||
|
|
@ -263,10 +267,10 @@ public static class EntityExtension
|
|||
others.Add(item);
|
||||
}
|
||||
|
||||
return new Tuple<IList<T>, IList<T>>(updates, others);
|
||||
return (updates, others);
|
||||
}
|
||||
|
||||
private static Int32 BatchSave<T>(IEntitySession session, IEnumerable<T> list) where T : IEntity
|
||||
private static Int32 BatchSave<T>(IEntitySession session, IEnumerable<T> list, BatchOption? option) where T : IEntity
|
||||
{
|
||||
// 没有其它唯一索引，且主键为空时，走批量插入
|
||||
var rs = 0;
|
||||
|
|
@ -289,18 +293,18 @@ public static class EntityExtension
|
|||
}
|
||||
list = upserts;
|
||||
|
||||
if (inserts.Count > 0) rs += BatchInsert(inserts, option: null, session);
|
||||
if (inserts.Count > 0) rs += BatchInsert(inserts, option, session);
|
||||
if (updates.Count > 0)
|
||||
{
|
||||
// 只有Oracle支持批量Update
|
||||
if (session.Dal.DbType == DatabaseType.Oracle)
|
||||
rs += BatchUpdate(updates, null, session);
|
||||
rs += BatchUpdate(updates, option, session);
|
||||
else
|
||||
upserts.AddRange(upserts);
|
||||
}
|
||||
}
|
||||
|
||||
if (list.Any()) rs += BatchUpsert(list, null, session);
|
||||
if (list.Any()) rs += BatchUpsert(list, option, session);
|
||||
|
||||
return rs;
|
||||
}
|
||||
|
|
@ -1408,7 +1412,8 @@ public static class EntityExtension
|
|||
for (var i = 0; i < fields.Length && i < line.Length; i++)
|
||||
{
|
||||
var fi = fields[i];
|
||||
if (fi != null && !line[i].IsNullOrEmpty()) entity.SetItem(fi.Name, line[i].ChangeType(fi.Type));
|
||||
//if (fi != null && !line[i].IsNullOrEmpty()) entity.SetItem(fi.Name, line[i].ChangeType(fi.Type));
|
||||
if (fi != null && !line[i].IsNullOrEmpty()) entity[fi.Name] = line[i].ChangeType(fi.Type);
|
||||
}
|
||||
|
||||
yield return entity;
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ using NewLife.Caching;
|
|||
using NewLife.Data;
|
||||
using NewLife.Log;
|
||||
using NewLife.Reflection;
|
||||
using XCode.Model;
|
||||
using XCode.Transform;
|
||||
|
||||
namespace XCode.Membership;
|
||||
|
|
@ -1060,7 +1061,11 @@ public partial class Area : Entity<Area>
|
|||
}
|
||||
}
|
||||
|
||||
bs.Save(true);
|
||||
//Meta.Factory.FullInsert = true;
|
||||
//count = bs.Save(true);
|
||||
count = bs.BatchUpsert(new BatchOption { FullInsert = true });
|
||||
|
||||
XTrace.WriteLine("合并三级地址成功：{0:n0}", count);
|
||||
|
||||
return count;
|
||||
}
|
||||
|
|
@ -1076,6 +1081,7 @@ public partial class Area : Entity<Area>
|
|||
|
||||
// 一次性加载四级地址
|
||||
var rs = FindAll(_.ID < 99_99_99_999);
|
||||
var bs = new List<Area>();
|
||||
|
||||
var count = 0;
|
||||
foreach (var r in list)
|
||||
|
|
@ -1097,7 +1103,8 @@ public partial class Area : Entity<Area>
|
|||
r.CreateTime = DateTime.Now;
|
||||
r.UpdateTime = DateTime.Now;
|
||||
r.Valid(DataMethod.Insert);
|
||||
r.SaveAsync();
|
||||
//r.SaveAsync();
|
||||
bs.Add(r);
|
||||
|
||||
count++;
|
||||
}
|
||||
|
|
@ -1121,13 +1128,20 @@ public partial class Area : Entity<Area>
|
|||
|
||||
XTrace.Log.Debug(re.Dirtys.Join(",", e => $"{e}={r2[e]}"));
|
||||
|
||||
r2.SaveAsync();
|
||||
//r2.SaveAsync();
|
||||
bs.Add(r2);
|
||||
|
||||
count++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//Meta.Factory.FullInsert = true;
|
||||
//count = bs.Save(true);
|
||||
count = bs.BatchUpsert(new BatchOption { FullInsert = true });
|
||||
|
||||
XTrace.WriteLine("合并四级地址成功：{0:n0}", count);
|
||||
|
||||
return count;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -86,7 +86,12 @@ public class UserModule : EntityModule
|
|||
|
||||
// 新增时如果没有当前用户，尝试使用环境变量中的用户名
|
||||
if (user == null && method == DataMethod.Insert)
|
||||
user = new User { Name = Environment.UserName };
|
||||
{
|
||||
// 如果用户名为空，或者是root或Administrator，则使用计算机名
|
||||
var name = Environment.UserName;
|
||||
if (name.IsNullOrEmpty() || name.EqualIgnoreCase("root", "Administrator")) name = Environment.MachineName;
|
||||
user = new User { Name = name };
|
||||
}
|
||||
|
||||
if (user != null)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ using System.IO;
|
|||
using System.Linq;
|
||||
using System.Net.Http;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using NewLife;
|
||||
using NewLife.Http;
|
||||
using NewLife.IP;
|
||||
|
|
@ -199,7 +200,7 @@ public class AreaTests
|
|||
}
|
||||
|
||||
[Fact]
|
||||
public async void Download()
|
||||
public async Task Download()
|
||||
{
|
||||
//var url = "http://www.mca.gov.cn/article/sj/xzqh/2020/2020/2020092500801.html";
|
||||
var url = "http://x.newlifex.com/202301xzqh.html";
|
||||
|
|
@ -214,7 +215,7 @@ public class AreaTests
|
|||
}
|
||||
|
||||
[Fact]
|
||||
public async void Download2024()
|
||||
public async Task Download2024()
|
||||
{
|
||||
//var url = "http://www.mca.gov.cn/article/sj/xzqh/2020/2020/2020092500801.html";
|
||||
//var url = "http://x.newlifex.com/202301xzqh.html";
|
||||
|
|
@ -242,6 +243,7 @@ public class AreaTests
|
|||
var rs = Parse(txt).ToList();
|
||||
Assert.NotNull(rs);
|
||||
Assert.True(rs.Count > 3000);
|
||||
Assert.Equal(3208, rs.Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
|
|
@ -257,6 +259,7 @@ public class AreaTests
|
|||
var rs = Parse(txt).ToList();
|
||||
Assert.NotNull(rs);
|
||||
Assert.True(rs.Count > 3000);
|
||||
Assert.Equal(3213, rs.Count);
|
||||
}
|
||||
|
||||
//[Fact]
|
||||
|
|
@ -320,7 +323,7 @@ public class AreaTests
|
|||
|
||||
[TestOrder(0)]
|
||||
[Fact]
|
||||
public async void Import()
|
||||
public async Task Import()
|
||||
{
|
||||
Area.Meta.Session.Dal.Db.ShowSQL = false;
|
||||
|
||||
|
|
@ -337,10 +340,12 @@ public class AreaTests
|
|||
Area.Meta.Session.Truncate();
|
||||
var rs = Area.Import(file, true, 3, false);
|
||||
Assert.Equal(3639, rs);
|
||||
Assert.Equal(3639, Area.FindCount());
|
||||
|
||||
Area.Meta.Session.Truncate();
|
||||
rs = Area.Import(file, true, 4, true);
|
||||
Assert.Equal(46533, rs);
|
||||
Assert.Equal(46533, Area.FindCount());
|
||||
}
|
||||
|
||||
Area.Meta.Session.Dal.Db.ShowSQL = true;
|
||||
|
|
|
|||
Loading…
Reference in New Issue